[llvm-branch-commits] [llvm] [AMDGPU] Lower loads and stores for address space 13 (PR #209541)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 3 09:50:13 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/209541
>From de15796db1dc06821eaba6f142b27fda004f36a0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 14 Jul 2026 00:06:53 -0500
Subject: [PATCH 01/12] Lower loads and stores for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 9 +
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 108 ++++
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 3 +
.../AMDGPU/AMDGPUInstructionSelector.cpp | 27 +
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 100 ++++
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 100 +++-
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 55 +++
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 71 +++
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 7 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 15 +
.../Target/AMDGPU/AMDGPUSearchableTables.td | 17 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 +
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 7 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 2 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 117 +++++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 3 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 73 ++-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 11 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 100 ++++
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 11 +
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 13 +
.../AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll | 384 +++++++++++++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 461 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 130 +++++
.../AddressSpaceVGPR/as-vgpr-inttoptr.ll | 48 ++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 31 ++
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
30 files changed, 1898 insertions(+), 24 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c72fa69aa1419..ad9cc6aa12c1d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -223,6 +223,9 @@ extern char &AMDGPURegBankLegalizeID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
+void initializeAMDGPUAssignIdxToM0LegacyPass(PassRegistry &);
+extern char &AMDGPUAssignIdxToM0ID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -437,6 +440,12 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
+class AMDGPUAssignIdxToM0Pass : public PassInfoMixin<AMDGPUAssignIdxToM0Pass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
new file mode 100644
index 0000000000000..8c5de37eb46e5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -0,0 +1,108 @@
+//===- AMDGPUAssignIdxToM0.cpp - Copy VGPR-memory indices to M0 ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Copy the register index of a VGPR "as memory" (address space 13)
+/// V_LOAD_IDX / V_STORE_IDX pseudo into M0, which V_MOVREL[SD] reads when the
+/// pseudo is lowered (see AMDGPULowerVGPREncoding). This runs before register
+/// allocation so the copy to M0 is inserted while the index is still virtual.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-assign-idx-to-m0"
+
+static bool assignIdxToM0(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.hasMovrel())
+ return false;
+
+ const SIInstrInfo *TII = ST.getInstrInfo();
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
+ if (!LdSt)
+ continue;
+
+ MachineOperand &IdxOp = LdSt->getIdxOp();
+ if (!IdxOp.isReg())
+ continue;
+
+ assert(!MI.isBundled());
+
+ // Remove the implicit-def $m0 that instruction selection added (to pin a
+ // divergent access inside its waterfall loop); M0 is written for real
+ // below.
+ int DefIdx = MI.findRegisterDefOperandIdx(AMDGPU::M0, /*TRI=*/nullptr);
+ assert(DefIdx >= 0);
+ MI.removeOperand(DefIdx);
+
+ // Add a copy from the index register to M0 and rewrite MI to read M0.
+ BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
+ .add(IdxOp);
+ IdxOp.setReg(AMDGPU::M0);
+ IdxOp.setIsKill();
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
+
+namespace {
+
+class AMDGPUAssignIdxToM0Legacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUAssignIdxToM0Legacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return assignIdxToM0(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Assign Idx To M0"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPUAssignIdxToM0Pass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!assignIdxToM0(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPUAssignIdxToM0Legacy::ID = 0;
+
+char &llvm::AMDGPUAssignIdxToM0ID = AMDGPUAssignIdxToM0Legacy::ID;
+
+INITIALIZE_PASS(AMDGPUAssignIdxToM0Legacy, DEBUG_TYPE,
+ "AMDGPU Assign Idx To M0", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index a88322df79036..38a692a45fa29 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -282,6 +282,9 @@ def : GINodeEquiv<G_AMDGPU_FMINIMUM3, AMDGPUfminimum3>;
def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_USHORT, SIbuffer_load_ushort>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 173d8a0c2b8ef..864a9ff82211d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPURegisterBankInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
@@ -4597,6 +4598,29 @@ bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
return true;
}
+bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
+ // Remember where the selected machine instruction will land.
+ MachineBasicBlock::iterator II = std::next(I.getIterator());
+
+ if (!selectImpl(I, *CoverageInfo))
+ return false;
+
+ // On a movrel subtarget the selected V_LOAD_IDX / V_STORE_IDX expands to an
+ // M0-relative move (see AMDGPUAssignIdxToM0 and AMDGPULowerVGPREncoding). Add
+ // an implicit-def of $m0: it records that the eventual move clobbers M0, and
+ // - because an instruction defining a physical register is not hoisted/sunk -
+ // keeps a divergent access pinned inside its waterfall loop.
+ // AMDGPUAssignIdxToM0 removes it when it writes M0 for real.
+ if (!Subtarget->hasMovrel())
+ return true;
+
+ auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
+ if (LdStIdx->getIdxOp().isReg())
+ LdStIdx->addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
+ return true;
+}
+
bool AMDGPUInstructionSelector::select(MachineInstr &I) {
if (!I.isPreISelOpcode()) {
@@ -4732,6 +4756,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
return selectBVHIntersectRayIntrinsic(I);
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ return selectRegLoadStore(I);
case AMDGPU::G_SBFX:
case AMDGPU::G_UBFX:
return selectG_SBFX_UBFX(I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index bd8ec0a769398..7364db759ce4b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -91,6 +91,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
bool selectCOPY_VCC_SCC(MachineInstr &I) const;
bool selectReadAnyLane(MachineInstr &I) const;
bool selectPHI(MachineInstr &I) const;
+ bool selectRegLoadStore(MachineInstr &I) const;
bool selectG_TRUNC(MachineInstr &I) const;
bool selectG_SZA_EXT(MachineInstr &I) const;
bool selectG_FPEXT(MachineInstr &I) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2180ce67d576f..2e666845e5637 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -16,6 +16,7 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUTargetMachine.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -428,6 +429,8 @@ static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
// register bank/uniformity and if the memory is invariant or not written in a
// kernel.
return IsLoad ? 512 : 128;
+ case AMDGPUAS::VGPR:
+ return 1024;
default:
// FIXME: Flat addresses may contextually need to be split to 32-bit parts
// if they may alias scratch depending on the subtarget. This needs to be
@@ -544,10 +547,23 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
+ // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
+ // to G_AMDGPU_REG_LOAD/STORE.
+ if (Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR)
+ return false;
return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
+// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
+// memory access producing/consuming a ValSize-bit value. Only whole-dword
+// accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported
+// for now; sub-dword (8/16-bit) support lands later.
+static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+ return MemSize == ValSize &&
+ AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+}
+
/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
@@ -1655,6 +1671,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// inserting addrspacecasts.
Actions.customIf(typeIs(1, Constant32Ptr));
+ // VGPR ("as memory") accesses are custom-lowered to the legal
+ // G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
+ // so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
+ // than failing to legalize.
+ Actions.customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
+ });
+
// Turn any illegal element vectors into something easier to deal
// with. These will ultimately produce 32-bit scalar shifts to extract the
// parts anyway.
@@ -3449,6 +3473,75 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
+/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into a legal
+/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE indexed by the pointer's dword offset
+/// (pointer >> 2). Parallels the SelectionDAG LowerLoadStoreVGPR.
+static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *B.getMRI();
+ MachineMemOperand &MMO = **MI.memoperands_begin();
+
+ const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ Register ValReg = MI.getOperand(0).getReg();
+ Register PtrReg = MI.getOperand(1).getReg();
+
+ const LLT ValTy = MRI.getType(ValReg);
+ const unsigned ValSize = ValTy.getSizeInBits();
+ // The GISel selection patterns for the indexed pseudos - and for the shift /
+ // readfirstlane that compute the index - match the extended integer LLT, so
+ // build the dword index (and the normalized register value below) with
+ // integer types rather than plain scalars.
+ const LLT I32 = LLT::integer(32);
+
+ // Only whole-dword, non-extending/non-truncating accesses are implemented.
+ // Reject anything else with a diagnostic instead of failing to legalize
+ // (sub-dword support lands in a later change).
+ if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
+ ValSize)) {
+ const Function &F = B.getMF().getFunction();
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F,
+ "unsupported access of VGPR 'as memory' address space (13); only "
+ "whole-dword loads and stores are implemented",
+ MI.getDebugLoc()));
+ if (!IsStore)
+ B.buildUndef(ValReg);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ // Normalize the value to i32 / <N x i32> so a selection pattern always
+ // exists (e.g. for v4i8).
+ LLT RegTy = ValTy;
+ if (ValTy.getScalarSizeInBits() != 32) {
+ unsigned NumDwords = ValSize / 32;
+ RegTy = NumDwords == 1 ? I32 : LLT::fixed_vector(NumDwords, I32);
+ }
+
+ if (IsStore) {
+ Register Value = ValReg;
+ if (RegTy != ValTy)
+ Value = B.buildBitcast(RegTy, Value).getReg(0);
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE, {}, {Value, Index.getReg(0)})
+ .addMemOperand(&MMO);
+ } else if (RegTy == ValTy) {
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
+ .addMemOperand(&MMO);
+ } else {
+ const auto Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
+ .addMemOperand(&MMO);
+ B.buildBitcast(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+}
+
bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
@@ -3459,6 +3552,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
+ if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ return lowerLoadStoreVGPR(Helper, MI);
+
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
@@ -3547,6 +3643,10 @@ bool AMDGPULegalizerInfo::legalizeStore(LegalizerHelper &Helper,
Register DataReg = MI.getOperand(0).getReg();
LLT DataTy = MRI.getType(DataReg);
+ if (MRI.getType(MI.getOperand(1).getReg()).getAddressSpace() ==
+ AMDGPUAS::VGPR)
+ return lowerLoadStoreVGPR(Helper, MI);
+
if (hasBufferRsrcWorkaround(DataTy)) {
Observer.changingInstr(MI);
castBufferRsrcArgToV4I32(MI, B, 0);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index f6c981ad464ab..852f53ed4c0df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -42,9 +42,11 @@
#include "AMDGPULowerVGPREncoding.h"
#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
#include "GCNSubtarget.h"
#include "SIDefines.h"
#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/bit.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
@@ -131,6 +133,7 @@ class AMDGPULowerVGPREncoding {
bool run(MachineFunction &MF);
private:
+ const GCNSubtarget *ST;
const SIInstrInfo *TII;
const SIRegisterInfo *TRI;
@@ -176,6 +179,13 @@ class AMDGPULowerVGPREncoding {
/// Handle single \p MI. \return true if changed.
bool runOnMachineInstr(MachineInstr &MI);
+ /// Lower a VGPR "as memory" (address space 13) indexed load/store pseudo
+ /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of M0-relative moves
+ /// (v_movrels_b32 for loads, v_movreld_b32 for stores) over the wave's vector
+ /// registers. M0 must already hold the dword index (see AMDGPUAssignIdxToM0).
+ /// This replaces the pseudo, which is erased.
+ void lowerLoadStoreIdx(MachineInstr &MI);
+
/// Compute the mode for a single \p MI given \p Ops operands
/// bit mapping. Optionally takes second array \p Ops2 for VOPD.
/// If provided and an operand from \p Ops is not a VGPR, then \p Ops2
@@ -378,6 +388,63 @@ bool AMDGPULowerVGPREncoding::runOnMachineInstr(MachineInstr &MI) {
return false;
}
+void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
+ auto &LdSt = cast<AMDGPUMI::VLoadStoreIdxInst>(MI);
+ MachineBasicBlock &BB = *MI.getParent();
+ const DebugLoc &DL = MI.getDebugLoc();
+ const bool IsStore = LdSt.mayStore();
+
+ // $data is operand 0 of both the load (def) and store (use) pseudos; M0
+ // already holds the dword index (see AMDGPUAssignIdxToM0).
+ Register Data = LdSt.getDataOp().getReg();
+ unsigned Offset = LdSt.getOffsetOp().getImm();
+ unsigned NumDwords = LdSt.getBitWidth() / 32;
+
+ // A statically out-of-range dword offset would fold into a base VGPR outside
+ // the addressable register file - an out-of-bounds access of the VGPR
+ // "as memory" (address space 13) region. When the whole file is addressable
+ // the index is allowed to wrap; otherwise it must stay in range.
+#ifndef NDEBUG
+ unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
+ MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
+ bool AllowOffsetWrap =
+ NumAddressableVGPRs == AMDGPU::IsaInfo::getTotalNumVGPRs(*ST);
+ assert((AllowOffsetWrap || Offset + NumDwords <= NumAddressableVGPRs) &&
+ "out of bounds VGPR 'as memory' (address space 13) access");
+#endif
+
+ unsigned Opcode =
+ IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+
+ // The dword index is (M0 + $offset). Fold $offset into the base register so
+ // each dword i reads/writes VGPR($offset + i) relative to M0.
+ for (unsigned i = 0; i < NumDwords; ++i) {
+ Register Base = AMDGPU::VGPR0 + Offset + i;
+ Register Sub = Data;
+ if (NumDwords != 1)
+ Sub = TRI->getSubReg(Data, TRI->getSubRegFromChannel(i));
+
+ MachineInstr *Mov;
+ if (IsStore)
+ Mov = BuildMI(BB, MI, DL, TII->get(Opcode))
+ .addReg(Base, RegState::Undef)
+ .addReg(Sub)
+ .getInstr();
+ else
+ Mov = BuildMI(BB, MI, DL, TII->get(Opcode), Sub)
+ .addReg(Base, RegState::Undef)
+ .getInstr();
+
+ // On subtargets with more than 256 addressable VGPRs the referenced
+ // register may need high address bits; reuse the S_SET_VGPR_MSB machinery
+ // to encode them. This is a no-op on movrel-only (<=256 VGPR) subtargets.
+ if (ST->has1024AddressableVGPRs())
+ runOnMachineInstr(*Mov);
+ }
+
+ MI.eraseFromParent();
+}
+
MachineBasicBlock::instr_iterator
AMDGPULowerVGPREncoding::handleClause(MachineBasicBlock::instr_iterator I) {
if (!ClauseRemaining)
@@ -551,12 +618,14 @@ bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
}
bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!ST.has1024AddressableVGPRs())
- return false;
+ ST = &MF.getSubtarget<GCNSubtarget>();
+ TII = ST->getInstrInfo();
+ TRI = ST->getRegisterInfo();
- TII = ST.getInstrInfo();
- TRI = ST.getRegisterInfo();
+ // The S_SET_VGPR_MSB encoding is only required on subtargets with more than
+ // 256 addressable VGPRs (gfx1250). On movrel-only subtargets the pass still
+ // runs, but only to lower the VGPR "as memory" indexed load/store pseudos.
+ const bool LowerVGPRMSBs = ST->has1024AddressableVGPRs();
LLVM_DEBUG(dbgs() << "*** AMDGPULowerVGPREncoding on " << MF.getName()
<< " ***\n");
@@ -573,6 +642,19 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
<< ":\n");
for (auto &MI : llvm::make_early_inc_range(MBB.instrs())) {
+ // Lower VGPR "as memory" indexed load/store pseudos on any subtarget that
+ // reaches this pass (movrel-only or gfx1250). This replaces the pseudo.
+ if (isa<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ lowerLoadStoreIdx(MI);
+ Changed = true;
+ continue;
+ }
+
+ // The remaining work only inserts VGPR MSB encoding, which is unnecessary
+ // on movrel-only subtargets.
+ if (!LowerVGPRMSBs)
+ continue;
+
if (MI.isMetaInstruction())
continue;
@@ -605,7 +687,7 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
}
if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
- ST.hasSetregVGPRMSBFixup()) {
+ ST->hasSetregVGPRMSBFixup()) {
Changed |= handleSetregMode(MI);
continue;
}
@@ -629,8 +711,10 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
}
// Reset the mode if we are falling through.
- LLVM_DEBUG(dbgs() << " end of BB, resetting mode\n");
- resetMode(MBB.instr_end());
+ if (LowerVGPRMSBs) {
+ LLVM_DEBUG(dbgs() << " end of BB, resetting mode\n");
+ resetMode(MBB.instr_end());
+ }
}
return Changed;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
new file mode 100644
index 0000000000000..697c63e2079d1
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -0,0 +1,55 @@
+//===-- AMDGPUMachineInstrs.cpp -*- C++ -*---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Convenience wrappers and helpers for AMDGPU-specific machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUMachineInstrs.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+using namespace AMDGPUMI;
+
+unsigned VLoadStoreIdxInst::getBitWidth() const {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ if (!Info)
+ llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
+ return Info->BitWidth;
+}
+
+int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
+ if (!Info)
+ return -1;
+ return Info->Opcode;
+}
+
+unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
+ int Opcode = tryGetOpcodeForBitWidth(Bits);
+ assert(Opcode != -1);
+ return Opcode;
+}
+
+int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
+ if (!Info)
+ return -1;
+ return Info->Opcode;
+}
+
+unsigned VStoreIdxInst::getOpcodeForBitWidth(unsigned Bits) {
+ int Opcode = tryGetOpcodeForBitWidth(Bits);
+ assert(Opcode != -1);
+ return Opcode;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
new file mode 100644
index 0000000000000..24e24aab53152
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -0,0 +1,71 @@
+//===-- AMDGPUMachineInstrs.h -*- C++ -*-----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Convenience wrappers and helpers for AMDGPU-specific machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
+
+#include "SIInstrInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+
+namespace llvm {
+namespace AMDGPUMI {
+
+// Wrapper for the whole-dword VGPR "as memory" (address space 13) indexed
+// load/store pseudos (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>). Operand layout:
+// load: (outs data), (ins idx, offset)
+// store: (outs), (ins data, idx, offset)
+// so data/idx/offset are always operands 0/1/2.
+class VLoadStoreIdxInst : public MachineInstr {
+public:
+ MachineOperand &getDataOp() { return getOperand(0); }
+ MachineOperand &getIdxOp() { return getOperand(1); }
+ MachineOperand &getOffsetOp() { return getOperand(2); }
+ const MachineOperand &getDataOp() const { return getOperand(0); }
+ const MachineOperand &getIdxOp() const { return getOperand(1); }
+ const MachineOperand &getOffsetOp() const { return getOperand(2); }
+
+ unsigned getBitWidth() const;
+
+ static bool classof(const MachineInstr *MI) {
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ }
+};
+
+class VLoadIdxInst : public VLoadStoreIdxInst {
+public:
+ static int tryGetOpcodeForBitWidth(unsigned Bits);
+ static unsigned getOpcodeForBitWidth(unsigned Bits);
+
+ static bool classof(const MachineInstr *MI) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ return Info && !Info->IsStore;
+ }
+};
+
+class VStoreIdxInst : public VLoadStoreIdxInst {
+public:
+ static int tryGetOpcodeForBitWidth(unsigned Bits);
+ static unsigned getOpcodeForBitWidth(unsigned Bits);
+
+ static bool classof(const MachineInstr *MI) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ return Info && Info->IsStore;
+ }
+};
+
+} // end namespace AMDGPUMI
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d052f3c73920c..d0255025298d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -113,6 +113,7 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
#ifndef MACHINE_FUNCTION_PASS
#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
#endif
+MACHINE_FUNCTION_PASS("amdgpu-assign-idx-to-m0", AMDGPUAssignIdxToM0Pass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7f7225ad6d311..6c426c6da11e8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1380,6 +1380,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
{{VgprV2S16},
{VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
+ // VGPR ("as memory") indexed load/store: the data is a VGPR value of any
+ // register class; the dword index is made uniform (waterfall) as an SGPR.
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD}).Any({{BRC}, {{VgprBRC}, {Sgpr32_WF}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 200234c23c886..5bdf0768179ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3087,6 +3087,14 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE: {
+ // The dword index (operand 1) must be uniform; a divergent index needs a
+ // waterfall loop.
+ applyDefaultMapping(OpdMapper);
+ executeInWaterfallLoop(B, MI, {1});
+ return;
+ }
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
@@ -4486,6 +4494,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
}
break;
}
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE: {
+ // data/result is a VGPR value; the dword index is uniform (SGPR).
+ OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+ OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ break;
+ }
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 2f7f72bd58f0e..abb38d86468e6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -431,3 +431,20 @@ def AMDGPUImageDMaskIntrinsicTable : GenericTable {
let PrimaryKeyName = "getAMDGPUImageDMaskIntrinsic";
let PrimaryKeyEarlyOut = 1;
}
+
+//===----------------------------------------------------------------------===//
+// V_LOAD/STORE_IDX opcode mapping table.
+//===----------------------------------------------------------------------===//
+
+def VLdStIdxOpcodeInfoTable : GenericTable {
+ let FilterClass = "VLdStIdxOpcodeInfo";
+ let CppTypeName = "VLdStIdxOpcodeInfo";
+ let Fields = ["Opcode", "BitWidth", "IsStore"];
+ let PrimaryKey = ["Opcode"];
+ let PrimaryKeyName = "getVLdStIdxOpcodeInfoByOpcodeImpl";
+}
+
+def getVLdStIdxOpcodeInfoByKeyImpl : SearchIndex {
+ let Table = VLdStIdxOpcodeInfoTable;
+ let Key = ["BitWidth", "IsStore"];
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8c2732630e2e6..a5dadbf76675a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -668,6 +668,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizePass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
+ initializeAMDGPUAssignIdxToM0LegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1779,6 +1780,10 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Copy the VGPR "as memory" load/store index into M0 before register
+ // allocation; the movrel emitted later by AMDGPULowerVGPREncoding reads it.
+ addPass(&AMDGPUAssignIdxToM0ID);
+
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
}
@@ -2595,6 +2600,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
+ // Set up M0 for the movrel that expands a VGPR "as memory" indexed access.
+ // Run before allocation so the index computation coalesces into M0.
+ addMachineFunctionPass(AMDGPUAssignIdxToM0Pass(), PMW);
+
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 5125735faddd6..c90498d02e4d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1122,13 +1122,16 @@ bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
// Loads from the private and flat address spaces are divergent, because
// threads can execute the load instruction with the same inputs and get
- // different results.
+ // different results. The same is true of the VGPR ("as memory") address
+ // space: it is a per-lane view of the vector registers, so an access at a
+ // uniform offset still yields a per-lane (divergent) value.
//
// All other loads are not divergent, because if threads issue loads with the
// same arguments, they will always get the same result.
if (const LoadInst *Load = dyn_cast<LoadInst>(V))
return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
- Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
+ Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
+ Load->getPointerAddressSpace() == AMDGPUAS::VGPR;
// Atomics are divergent because they are executed sequentially: when an
// atomic operation refers to the same address in each thread, then each
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..054fc1d14d7cb 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -47,6 +47,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUArgumentUsageInfo.cpp
AMDGPUAsanInstrumentation.cpp
AMDGPUAsmPrinter.cpp
+ AMDGPUAssignIdxToM0.cpp
AMDGPUAtomicOptimizer.cpp
AMDGPUAttributor.cpp
AMDGPUBarrierLatency.cpp
@@ -83,6 +84,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPULowerExecSync.cpp
AMDGPUSwLowerLDS.cpp
AMDGPUMachineFunctionInfo.cpp
+ AMDGPUMachineInstrs.cpp
AMDGPUMachineModuleInfo.cpp
AMDGPUMacroFusion.cpp
AMDGPUMCInstLower.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9cf33b4ccbb87..c2e0f0283e91e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUSelectionDAGInfo.h"
#include "AMDGPUTargetMachine.h"
@@ -13169,6 +13170,87 @@ static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
return true;
}
+// Lower a load or store of the VGPR ("as memory") address space (13) to a
+// REG_LOAD / REG_STORE target node. The 32-bit pointer is a byte offset into
+// the wave's view of its vector registers; the target node carries the dword
+// index (pointer >> 2). Recognizing a constant dword offset is left to the
+// selection patterns, which fold an (add index, imm) shape into the pseudo.
+//
+// TODO: sub-dword (8/16-bit) accesses are not yet supported; they are
+// diagnosed as unsupported below.
+SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ MemSDNode *MemOp = cast<MemSDNode>(Op);
+ EVT MemVT = MemOp->getMemoryVT();
+ unsigned BitWidth = MemVT.getSizeInBits();
+
+ // Only whole-dword, non-extending/non-truncating accesses are implemented.
+ // Reject anything else with a diagnostic (replacing the value with poison)
+ // instead of failing instruction selection. Both callers - operation
+ // legalization and the pre-ISel combine - replace the node with this result,
+ // so the diagnostic is emitted exactly once.
+ auto reportUnsupported = [&]() -> SDValue {
+ const Function &F = DAG.getMachineFunction().getFunction();
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ F,
+ "unsupported access of VGPR 'as memory' address space (13); only "
+ "whole-dword loads and stores are implemented",
+ DL.getDebugLoc()));
+ if (isa<StoreSDNode>(MemOp))
+ return MemOp->getChain();
+ return DAG.getMergeValues(
+ {DAG.getPOISON(Op.getValueType()), MemOp->getChain()}, DL);
+ };
+
+ if (BitWidth < 32)
+ return reportUnsupported();
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD)
+ return reportUnsupported();
+ if (AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(BitWidth) == -1)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore())
+ return reportUnsupported();
+ if (AMDGPUMI::VStoreIdxInst::tryGetOpcodeForBitWidth(BitWidth) == -1)
+ return reportUnsupported();
+ }
+
+ SDValue Chain = MemOp->getChain();
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, MemOp->getBasePtr(),
+ DAG.getConstant(2, DL, MVT::i32));
+
+ // View the access as i32 / <N x i32> when the memory type is not register
+ // legal (e.g. v4i8), bitcasting the value across.
+ EVT RegVT = MemVT;
+ if (!isTypeLegal(RegVT)) {
+ unsigned NumDwords = BitWidth / 32;
+ RegVT = NumDwords == 1
+ ? EVT(MVT::i32)
+ : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumDwords);
+ }
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ if (RegVT != MemVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, RegVT, Value);
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ SDValue NewLoad = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD, DL, DAG.getVTList(RegVT, MVT::Other), {Chain, Index},
+ MemVT, LoadOp->getMemOperand());
+ if (RegVT == MemVT)
+ return NewLoad;
+ SDValue Value = DAG.getNode(ISD::BITCAST, DL, MemVT, NewLoad);
+ return DAG.getMergeValues({Value, NewLoad.getValue(1)}, DL);
+}
+
SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
LoadSDNode *Load = cast<LoadSDNode>(Op);
@@ -13176,6 +13258,9 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
EVT MemVT = Load->getMemoryVT();
MachineMemOperand *MMO = Load->getMemOperand();
+ if (Load->getAddressSpace() == AMDGPUAS::VGPR)
+ return LowerLoadStoreVGPR(Op, DAG);
+
if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
if (MemVT == MVT::i16 && isTypeLegal(MVT::i16))
return SDValue();
@@ -13848,6 +13933,9 @@ SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
StoreSDNode *Store = cast<StoreSDNode>(Op);
EVT VT = Store->getMemoryVT();
+ if (Store->getAddressSpace() == AMDGPUAS::VGPR)
+ return LowerLoadStoreVGPR(Op, DAG);
+
if (VT == MVT::i1) {
return DAG.getTruncStore(
Store->getChain(), DL,
@@ -18657,6 +18745,19 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
if (auto Res = promoteUniformOpToI32(SDValue(N, 0), DCI))
return Res;
break;
+ case ISD::LOAD:
+ // Lower a VGPR ("as memory") address space (13) load to a REG_LOAD target
+ // node. Done here (not via operation legalization) so it also fires at -O0,
+ // where a scalar load is otherwise Legal and never reaches LowerLOAD.
+ if (cast<LoadSDNode>(N)->getAddressSpace() == AMDGPUAS::VGPR)
+ if (SDValue V = LowerLoadStoreVGPR(SDValue(N, 0), DCI.DAG))
+ return V;
+ break;
+ case ISD::STORE:
+ if (cast<StoreSDNode>(N)->getAddressSpace() == AMDGPUAS::VGPR)
+ if (SDValue V = LowerLoadStoreVGPR(SDValue(N, 0), DCI.DAG))
+ return V;
+ break;
default:
break;
}
@@ -19256,6 +19357,22 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
+ // A VGPR "as memory" indexed load/store with a register index expands (on
+ // movrel subtargets) to an M0-relative move. Add an implicit-def of $m0: it
+ // records that the eventual move clobbers M0, and - because an instruction
+ // defining a physical register is not hoisted/sunk - keeps a divergent access
+ // pinned inside its waterfall loop. AMDGPUAssignIdxToM0 removes this when it
+ // writes M0 for real.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (getSubtarget()->hasMovrel()) {
+ MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg())
+ MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
+ }
+ return;
+ }
+
if (TII->isImage(MI))
TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index a181fa072d534..fec7510522db8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -115,6 +115,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
+ // Lower a load/store of the VGPR ("as memory") address space (13) to a
+ // REG_LOAD/REG_STORE target node indexed by the pointer's dword offset.
+ SDValue LowerLoadStoreVGPR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFastUnsafeFDIV64(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 66e252a461cd3..f454f5ded2bec 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUMachineInstrs.h"
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
@@ -542,6 +543,18 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return true;
}
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ BaseOp = &LdStIdx->getIdxOp();
+ OffsetOp = &LdStIdx->getOffsetOp();
+
+ BaseOps.push_back(BaseOp);
+ Offset = OffsetOp->getImm() * 4; // Offset has units of dwords.
+
+ // Get appropriate operand, and compute width accordingly.
+ Width = LocationSize::precise(LdStIdx->getBitWidth() / 8);
+ return true;
+ }
+
return false;
}
@@ -4033,10 +4046,20 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
return false;
- if (isLDSDMA(MIa) || isLDSDMA(MIb))
+ if (MIa.isBundle() || MIb.isBundle())
return false;
- if (MIa.isBundle() || MIb.isBundle())
+ // VGPR "as memory" indexed accesses only alias each other, and then only
+ // when their [idx+offset, idx+offset+width) dword ranges overlap.
+ const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa);
+ const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb);
+ if (IsLdStIdxA || IsLdStIdxB) {
+ if (IsLdStIdxA && IsLdStIdxB)
+ return checkInstOffsetsDoNotOverlap(MIa, MIb);
+ return true;
+ }
+
+ if (isLDSDMA(MIa) || isLDSDMA(MIb))
return false;
// TODO: Should we check the address space from the MachineMemOperand? That
@@ -5658,6 +5681,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
+ const bool IsPreRA = !MI.getMF()->getProperties().hasNoVRegs();
const unsigned StaticNumOps =
Desc.getNumOperands() + Desc.implicit_uses().size();
@@ -5667,7 +5691,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// post RA scheduler where the main implicit operand is killed and
// implicit-defs are added for sub-registers that remain live after this
// instruction.
- if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
+ if (IsPreRA && MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
ErrInfo = "missing implicit register operands";
return false;
}
@@ -5680,20 +5704,22 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
unsigned UseOpIdx;
- if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
- UseOpIdx != StaticNumOps + 1) {
+ if (IsPreRA && (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
+ UseOpIdx != StaticNumOps + 1)) {
ErrInfo = "movrel implicit operands should be tied";
return false;
}
}
- const MachineOperand &Src0 = MI.getOperand(Src0Idx);
- const MachineOperand &ImpUse
- = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
- if (!ImpUse.isReg() || !ImpUse.isUse() ||
- !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
- ErrInfo = "src0 should be subreg of implicit vector use";
- return false;
+ if (IsPreRA) {
+ const MachineOperand &Src0 = MI.getOperand(Src0Idx);
+ const MachineOperand &ImpUse =
+ MI.getOperand(StaticNumOps + NumImplicitOps - 1);
+ if (!ImpUse.isReg() || !ImpUse.isUse() ||
+ !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
+ ErrInfo = "src0 should be subreg of implicit vector use";
+ return false;
+ }
}
}
@@ -7503,6 +7529,17 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
return CreatedBB;
}
+ // A VGPR "as memory" indexed load/store needs its dword index in an SGPR (it
+ // becomes M0). A divergent (VGPR) index is made uniform with a waterfall
+ // loop that executes the access once per unique index across the wave.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ MachineOperand *Idx = &LdStIdx->getIdxOp();
+ if (Idx->isReg() && Idx->getReg().isVirtual() &&
+ !RI.isSGPRClass(MRI.getRegClass(Idx->getReg())))
+ CreatedBB = generateWaterFallLoop(*this, MI, {Idx}, MDT);
+ return CreatedBB;
+ }
+
// Legalize PHI
// The register class of the operands must be the same type as the register
// class of the output.
@@ -10974,9 +11011,16 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
return ValueUniformity::Default;
}
+ // A VGPR ("as memory") indexed load is always divergent: it reads the wave's
+ // per-lane view of its vector registers, so even a uniform index yields a
+ // per-lane (divergent) value.
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ return ValueUniformity::NeverUniform;
+
// Loads from the private and flat address spaces are divergent, because
// threads can execute the load instruction with the same inputs and get
- // different results.
+ // different results. The VGPR address space is likewise divergent (see
+ // above; this covers a G_LOAD not yet legalized to G_AMDGPU_REG_LOAD).
//
// All other loads are not divergent, because if threads issue loads with the
// same arguments, they will always get the same result.
@@ -10987,7 +11031,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {
return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
- mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
+ mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS ||
+ mmo->getAddrSpace() == AMDGPUAS::VGPR;
})) {
// At least one MMO in a non-global address space.
return ValueUniformity::NeverUniform;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 7be067c085b5f..c6bbfc1ec14b4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -59,6 +59,17 @@ def GFX10Gen : GFXGen<isGFX10Only, "GFX10", "_gfx10", SIEncodingFamily.G
// modifier behavior with dx10_enable.
def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPUnaryOp>;
+// VGPR address space (13) load/store with a dword index operand. The index is
+// the byte offset into the wave's view of its vector registers, divided by 4.
+def SDTRegIdxLoad : SDTypeProfile<1, 1,
+ [SDTCisVT<1, i32>]>; // dword_index
+def SDTRegIdxStore : SDTypeProfile<0, 2,
+ [SDTCisVT<1, i32>]>; // data, dword_index
+def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d978735a92db8..765a5fc814d79 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1061,6 +1061,88 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
} // End Uses = [EXEC], Defs = [M0, EXEC]
+//===----------------------------------------------------------------------===//
+// VGPR "as memory" indexed load/store pseudos (address space 13)
+//===----------------------------------------------------------------------===//
+
+// V_LOAD_IDX_B<N> / V_STORE_IDX_B<N> load or store N bits from/to the wave's
+// view of its vector registers, at a dword index of ($idx + $offset). $idx is
+// a 32-bit value that may be uniform (SGPR) or divergent (VGPR); $offset is a
+// constant dword offset folded in at selection.
+//
+// AMDGPUAssignIdxToM0 copies $idx into M0 before register allocation, then
+// AMDGPULowerVGPREncoding lowers each into v_movrels_b32 (load) /
+// v_movreld_b32 (store) over the wave's vector registers.
+
+// Populates the VLdStIdxOpcodeInfo searchable table (see AMDGPUMachineInstrs.h
+// and AMDGPU::getVLdStIdxOpcodeInfo*), mapping each pseudo to its bit width and
+// load/store direction.
+class VLdStIdxOpcodeInfo<int size, bit isStore> {
+ Instruction Opcode = !cast<Instruction>(NAME);
+ bits<12> BitWidth = size;
+ bit IsStore = isStore;
+}
+
+foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
+ VReg_224, VReg_256, VReg_288, VReg_320, VReg_352, VReg_384,
+ VReg_512, VReg_1024] in {
+ // The units of $idx and $offset are in dwords.
+ //
+ // VALU adds an implicit $exec use; together with the implicit-def $m0 added
+ // by AdjustInstrPostInstrSelection (see hasPostISelHook), this keeps a
+ // divergent-index access pinned inside its waterfall loop rather than being
+ // hoisted/sunk out.
+ def V_LOAD_IDX_B#rc.Size : VPseudoInstSI <
+ (outs rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset)>,
+ VLdStIdxOpcodeInfo<rc.Size, 0> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let hasPostISelHook = 1;
+ }
+ def V_STORE_IDX_B#rc.Size : VPseudoInstSI <
+ (outs),
+ (ins rc:$data, SReg_32:$idx, i32imm:$offset)>,
+ VLdStIdxOpcodeInfo<rc.Size, 1> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let hasPostISelHook = 1;
+ }
+}
+
+// Select the REG_LOAD/REG_STORE target nodes into the sized indexed pseudos.
+// The pointer is a byte offset into the file; SIreg_load/SIreg_store carry the
+// dword index (ptr >> 2), and an (add idx, imm) shape folds a constant dword
+// offset into the pseudo's $offset operand.
+multiclass VRegIdxLoadStorePat<ValueType vt> {
+ defvar load_inst = !cast<Instruction>("V_LOAD_IDX_B"#vt.Size);
+ defvar store_inst = !cast<Instruction>("V_STORE_IDX_B"#vt.Size);
+
+ def : GCNPat<
+ (vt (SIreg_load (add i32:$idx, (i32 imm:$offset)))),
+ (load_inst $idx, imm:$offset)>;
+ def : GCNPat<
+ (vt (SIreg_load i32:$idx)),
+ (load_inst $idx, 0)>;
+ def : GCNPat<
+ (SIreg_store vt:$data, (add i32:$idx, (i32 imm:$offset))),
+ (store_inst $data, $idx, imm:$offset)>;
+ def : GCNPat<
+ (SIreg_store vt:$data, i32:$idx),
+ (store_inst $data, $idx, 0)>;
+}
+
+foreach vt = !listconcat(
+ Reg32Types.types, Reg64Types.types, Reg96Types.types, Reg128Types.types,
+ Reg160Types.types, Reg192Types.types, Reg224Types.types, Reg256Types.types,
+ Reg288Types.types, Reg320Types.types, Reg352Types.types, Reg384Types.types,
+ Reg512Types.types, Reg1024Types.types) in
+defm : VRegIdxLoadStorePat<vt>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4820,6 +4902,24 @@ def G_AMDGPU_BUFFER_STORE_FORMAT_D16 : BufferStoreGenericInstruction;
def G_AMDGPU_TBUFFER_STORE_FORMAT : TBufferStoreGenericInstruction;
def G_AMDGPU_TBUFFER_STORE_FORMAT_D16 : TBufferStoreGenericInstruction;
+// GlobalISel equivalents of the REG_LOAD / REG_STORE target nodes: a load or
+// store of the VGPR ("as memory") address space, indexed by the pointer's
+// dword offset. They select to the V_LOAD_IDX_B<N> / V_STORE_IDX_B<N> pseudos
+// through the same TableGen patterns (see GINodeEquiv in AMDGPUGISel.td).
+def G_AMDGPU_REG_LOAD : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 71d95a23e30d3..2370396a72932 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -478,9 +478,20 @@ struct FP4FP8DstByteSelInfo {
#define GET_getMFMA_F8F6F4_WithSize_IMPL
#define GET_isMFMA_F8F6F4Table_IMPL
#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
+#define GET_VLdStIdxOpcodeInfoTable_DECL
+#define GET_VLdStIdxOpcodeInfoTable_IMPL
#include "AMDGPUGenSearchableTables.inc"
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByOpcode(unsigned Opc) {
+ return getVLdStIdxOpcodeInfoByOpcodeImpl(Opc);
+}
+
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByKey(uint16_t BitWidth,
+ bool IsStore) {
+ return getVLdStIdxOpcodeInfoByKeyImpl(BitWidth, IsStore);
+}
+
int getMTBUFBaseOpcode(unsigned Opc) {
const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
return Info ? Info->BaseOpcode : -1;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 5e72eaee43846..23f5a309d8570 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -507,6 +507,19 @@ struct MIMGInfo {
LLVM_READONLY
const MIMGInfo *getMIMGInfo(unsigned Opc);
+struct VLdStIdxOpcodeInfo {
+ unsigned Opcode;
+ uint16_t BitWidth;
+ bool IsStore;
+};
+
+LLVM_READONLY
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByOpcode(unsigned Opc);
+
+LLVM_READONLY
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByKey(uint16_t BitWidth,
+ bool IsStore);
+
LLVM_READONLY
int getMTBUFBaseOpcode(unsigned Opc);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
new file mode 100644
index 0000000000000..08d3095de3561
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
@@ -0,0 +1,384 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of the VGPR "as memory" address space (13) on a
+; movrel-capable subtarget (gfx12). A load/store of a uniform (SGPR) pointer
+; lowers to an M0-relative move (v_movrels_b32 / v_movreld_b32) over the wave's
+; vector registers, with the dword index (pointer >> 2) placed in M0.
+
+define i32 @load_i32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define i64 @load_i64(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %p
+ %y = add i64 %x, 1
+ ret i64 %y
+}
+
+define <2 x float> @load_v2f32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_v2f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_add_f32 v0, 0x42280000, v0 :: v_dual_add_f32 v1, 0x42280000, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x float>, ptr addrspace(13) %p
+ %y = fadd <2 x float> %x, <float 42.0, float 42.0>
+ ret <2 x float> %y
+}
+
+define <3 x float> @load_v3f32(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_v3f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 64
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_v3f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 64
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %p.2 = getelementptr i32, ptr addrspace(13) %p, i32 16
+ %x = load <3 x float>, ptr addrspace(13) %p
+ %y = load <3 x float>, ptr addrspace(13) %p.2
+ %z = fadd <3 x float> %x, %y
+ ret <3 x float> %z
+}
+
+define void @store_i32(ptr addrspace(13) inreg %p, i32 %x) {
+; GFX12-LABEL: store_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i64(ptr addrspace(13) inreg %p, i64 %x, i64 %y) {
+; GFX12-LABEL: store_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %z = add i64 %x, %y
+ store i64 %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v3i32(ptr addrspace(13) inreg %p, <3 x i32> %x, <3 x i32> %y) {
+; GFX12-SDAG-LABEL: store_v3i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v3i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = add <3 x i32> %x, %y
+ store <3 x i32> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v8f16(ptr addrspace(13) inreg %p, <8 x half> %x, <8 x half> %y) {
+; GFX12-SDAG-LABEL: store_v8f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-SDAG-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-SDAG-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-SDAG-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v8f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-GISEL-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-GISEL-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = fadd <8 x half> %x, %y
+ store <8 x half> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @copy_i32(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %src
+ store i32 %x, ptr addrspace(13) %dst
+ ret void
+}
+
+define void @copy_v2i32_unaligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_v2i32_unaligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x i32>, ptr addrspace(13) %src, align 4
+ store <2 x i32> %x, ptr addrspace(13) %dst, align 4
+ ret void
+}
+
+define void @copy_i64_aligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i64_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %src, align 8
+ store i64 %x, ptr addrspace(13) %dst, align 8
+ ret void
+}
+
+; Null and poison pointers must be accepted (produce valid code) rather than
+; crash or fail the machine verifier. The specific null-pointer value is
+; defined by the parent change that introduces the address space.
+
+define i32 @load_null() {
+; GFX12-LABEL: load_null:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) null
+ ret i32 %x
+}
+
+define void @store_null(i32 %v) {
+; GFX12-LABEL: store_null:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 0
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) null
+ ret void
+}
+
+define i32 @load_poison() {
+; GFX12-SDAG-LABEL: load_poison:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_poison:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) poison
+ ret i32 %x
+}
+
+define void @store_poison(i32 %v) {
+; GFX12-SDAG-LABEL: store_poison:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_poison:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) poison
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
new file mode 100644
index 0000000000000..48d90ecaf2eff
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -0,0 +1,461 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Copy from VGPR "as memory" (address space 13) to global memory across the
+; range of legal whole-dword access sizes (32 up to 1024 bits). Each uniform
+; load lowers to a sequence of per-dword M0-relative moves (v_movrels_b32).
+
+define void @copy_i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %in
+ store i32 %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v2i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v2i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x i32>, ptr addrspace(13) %in
+ store <2 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v3i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v3i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-NEXT: global_store_b96 v3, v[0:2], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <3 x i32>, ptr addrspace(13) %in
+ store <3 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v4i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v4i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <4 x i32>, ptr addrspace(13) %in
+ store <4 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v5i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v5i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b32 v5, v4, s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v5i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b32 v5, v4, s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <5 x i32>, ptr addrspace(13) %in
+ store <5 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v6i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v6i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v6i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <6 x i32>, ptr addrspace(13) %in
+ store <6 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v7i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v7i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b96 v7, v[4:6], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v7, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v7i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v7, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b96 v7, v[4:6], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <7 x i32>, ptr addrspace(13) %in
+ store <7 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v8i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v8i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v8i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <8 x i32>, ptr addrspace(13) %in
+ store <8 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v16i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v16i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-SDAG-NEXT: s_clause 0x3
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:48
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v16i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-GISEL-NEXT: s_clause 0x3
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:48
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <16 x i32>, ptr addrspace(13) %in
+ store <16 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v32i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v32i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v16, v16
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v17, v17
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v18, v18
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v19, v19
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v20, v20
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v21, v21
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v22, v22
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v23, v23
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v24, v24
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v25, v25
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v26, v26
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v27, v27
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v28, v28
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v29, v29
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v30, v30
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v31, v31
+; GFX12-SDAG-NEXT: s_clause 0x7
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v32i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v16, v16
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v17, v17
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v18, v18
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v19, v19
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v20, v20
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v21, v21
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v22, v22
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v23, v23
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v24, v24
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v25, v25
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v26, v26
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v27, v27
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v28, v28
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v29, v29
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v30, v30
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v31, v31
+; GFX12-GISEL-NEXT: s_clause 0x7
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <32 x i32>, ptr addrspace(13) %in
+ store <32 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
new file mode 100644
index 0000000000000..28aa0ff7f0fb9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; A divergent (per-lane) dword index into VGPR "as memory" (address space 13)
+; is handled with a waterfall loop: for each unique index across the wave, set
+; M0 and do the M0-relative move under a matching-lane EXEC subset. The pointer
+; arrives in a VGPR (no inreg), so the index (pointer >> 2) is divergent.
+
+define i32 @load_i32(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define void @store_i32(ptr addrspace(13) %p, i32 %x) {
+; GFX12-SDAG-LABEL: store_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
new file mode 100644
index 0000000000000..393928d359f96
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; A constant VGPR "as memory" (address space 13) pointer formed with inttoptr:
+; the dword index (256 >> 2 = 64) is a compile-time constant, so M0 is set from
+; that constant and the access is a single M0-relative move.
+
+define i32 @load_i32() {
+; GFX12-LABEL: load_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 64
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %p = inttoptr i32 256 to ptr addrspace(13)
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define void @store_i32(i32 %x) {
+; GFX12-LABEL: store_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_mov_b32 m0, 64
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %p = inttoptr i32 256 to ptr addrspace(13)
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
new file mode 100644
index 0000000000000..40263e1c975ad
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -0,0 +1,31 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
+; not yet implemented. They must be rejected with a clean diagnostic on both
+; SelectionDAG and GlobalISel, rather than failing with "cannot select" /
+; "unable to legalize".
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 2f3bc3f259e89..435ea17e77122 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -69,6 +69,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-assign-idx-to-m0
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -217,6 +218,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-assign-idx-to-m0
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -405,6 +407,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-assign-idx-to-m0
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index d10df2ed1d11f..8a1be666cb68a 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Assign Idx To M0
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -351,6 +352,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -673,6 +675,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -999,6 +1002,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Assign Idx To M0
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1339,6 +1343,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Assign Idx To M0
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From aa2691034b3029a543a5a9eabf1592690819a089 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 15 Jul 2026 14:25:19 -0500
Subject: [PATCH 02/12] Waterfall any non-SGPR VGPR-memory index, not just
virtual
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f454f5ded2bec..f577f31911de4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7534,8 +7534,10 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
// loop that executes the access once per unique index across the wave.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
MachineOperand *Idx = &LdStIdx->getIdxOp();
- if (Idx->isReg() && Idx->getReg().isVirtual() &&
- !RI.isSGPRClass(MRI.getRegClass(Idx->getReg())))
+ // Waterfall any non-SGPR index. isSGPRReg handles both virtual and physical
+ // registers, so a physical (non-SGPR) index - not expected here, but still
+ // possible - is made uniform rather than silently skipped.
+ if (Idx->isReg() && !RI.isSGPRReg(MRI, Idx->getReg()))
CreatedBB = generateWaterFallLoop(*this, MI, {Idx}, MDT);
return CreatedBB;
}
>From 98f14b6e315c88659d5e44507e8f6afd17790eeb Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 15 Jul 2026 14:36:32 -0500
Subject: [PATCH 03/12] Do not set a kill flag on the M0 index copy in
AMDGPUAssignIdxToM0
---
llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index 8c5de37eb46e5..3bc6709b7c197 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -55,10 +55,11 @@ static bool assignIdxToM0(MachineFunction &MF) {
MI.removeOperand(DefIdx);
// Add a copy from the index register to M0 and rewrite MI to read M0.
+ // No kill flag is set on the M0 use: kill flags are deprecated and are a
+ // no-op on the reserved M0 register.
BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
.add(IdxOp);
IdxOp.setReg(AMDGPU::M0);
- IdxOp.setIsKill();
Changed = true;
}
}
>From 30811b87a78bdb71777f6dcf7253792c1d1e0ef2 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 16 Jul 2026 09:07:52 -0500
Subject: [PATCH 04/12] Mask the VGPR-memory movrel base into the addressable
range
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 852f53ed4c0df..86bda1ae9a4ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -400,13 +400,14 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
unsigned Offset = LdSt.getOffsetOp().getImm();
unsigned NumDwords = LdSt.getBitWidth() / 32;
- // A statically out-of-range dword offset would fold into a base VGPR outside
- // the addressable register file - an out-of-bounds access of the VGPR
- // "as memory" (address space 13) region. When the whole file is addressable
- // the index is allowed to wrap; otherwise it must stay in range.
-#ifndef NDEBUG
+ // A statically out-of-range dword offset is an out-of-bounds (undefined
+ // behavior) access of the VGPR "as memory" (address space 13) region. Rather
+ // than diagnose it or emit an invalid register, mask the base into the
+ // addressable VGPR range below so the access is accepted and verifier-clean,
+ // matching the downstream implementation.
unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
+#ifndef NDEBUG
bool AllowOffsetWrap =
NumAddressableVGPRs == AMDGPU::IsaInfo::getTotalNumVGPRs(*ST);
assert((AllowOffsetWrap || Offset + NumDwords <= NumAddressableVGPRs) &&
@@ -417,9 +418,11 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
// The dword index is (M0 + $offset). Fold $offset into the base register so
- // each dword i reads/writes VGPR($offset + i) relative to M0.
+ // each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
+ // into the addressable range so a statically out-of-bounds offset still
+ // resolves to a valid register.
for (unsigned i = 0; i < NumDwords; ++i) {
- Register Base = AMDGPU::VGPR0 + Offset + i;
+ Register Base = AMDGPU::VGPR0 + ((Offset + i) & (NumAddressableVGPRs - 1));
Register Sub = Data;
if (NumDwords != 1)
Sub = TRI->getSubReg(Data, TRI->getSubRegFromChannel(i));
>From fc6d17c2f40f5b82b44e37ab45f528e476b80863 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 28 Jul 2026 00:35:27 +0300
Subject: [PATCH 05/12] Simplify demanded bits
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 23 +++++++++
.../as-vgpr-index-demanded-bits.ll | 51 +++++++++++++++++++
3 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 86bda1ae9a4ea..90b29cafafe35 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -403,8 +403,7 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
// A statically out-of-range dword offset is an out-of-bounds (undefined
// behavior) access of the VGPR "as memory" (address space 13) region. Rather
// than diagnose it or emit an invalid register, mask the base into the
- // addressable VGPR range below so the access is accepted and verifier-clean,
- // matching the downstream implementation.
+ // addressable VGPR range below so the access is accepted and verifier-clean.
unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
#ifndef NDEBUG
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c2e0f0283e91e..b712ce06f6d5c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18880,6 +18880,29 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
return performInsertVectorEltCombine(N, DCI);
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
+ case AMDGPUISD::REG_LOAD:
+ case AMDGPUISD::REG_STORE: {
+ const SIMachineFunctionInfo *MFI =
+ DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
+ unsigned NumAddressableVGPRs =
+ Subtarget->getAddressableNumVGPRs(MFI->getDynamicVGPRBlockSize());
+ APInt IndexMask =
+ APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
+
+ unsigned IndexOpIdx = 0;
+ switch (N->getOpcode()) {
+ case AMDGPUISD::REG_LOAD:
+ IndexOpIdx = 1;
+ break;
+ case AMDGPUISD::REG_STORE:
+ IndexOpIdx = 2;
+ break;
+ }
+
+ if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
+ return SDValue(N, 0);
+ break;
+ }
case ISD::LOAD: {
if (SDValue Widened = widenLoad(cast<LoadSDNode>(N), DCI))
return Widened;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
new file mode 100644
index 0000000000000..1a2689f23a1c4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; The VGPR "as memory" (address space 13) dword index only needs enough bits to
+; address all addressable VGPRs, so a redundant high-bit mask feeding the index
+; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
+; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
+; path the mask must not survive into the M0 index computation.
+
+define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
+; GFX12-SDAG-LABEL: load_masked_index:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_masked_index:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %idx = and i32 %arg, 65535
+ %ptr = inttoptr i32 %idx to ptr addrspace(13)
+ %v = load i32, ptr addrspace(13) %ptr
+ ret i32 %v
+}
+
+define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
+; GFX12-SDAG-LABEL: store_masked_index:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: store_masked_index:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %idx = and i32 %arg, 65535
+ %ptr = inttoptr i32 %idx to ptr addrspace(13)
+ store i32 %val, ptr addrspace(13) %ptr
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From 5a58959ccbee5f1e96e60db0fc54418f2384a0b3 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:56:31 +0300
Subject: [PATCH 06/12] Run AMDGPUAssignIdxToM0 for optnone functions
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 5 ++-
.../AddressSpaceVGPR/as-vgpr-optnone.ll | 43 +++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index 3bc6709b7c197..f2ddf2b5d9bb1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -76,8 +76,9 @@ class AMDGPUAssignIdxToM0Legacy : public MachineFunctionPass {
AMDGPUAssignIdxToM0Legacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: without the copy to M0
+ // the movrel that AMDGPULowerVGPREncoding emits later reads a stale index.
+ // It therefore must not be skipped for optnone functions.
return assignIdxToM0(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
new file mode 100644
index 0000000000000..cd48756741766
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+
+; AMDGPUAssignIdxToM0 is required lowering rather than an optimization: the
+; v_movrel that AMDGPULowerVGPREncoding emits reads the dword index from M0, so
+; without the copy to M0 it reads a stale value. The pass must therefore run
+; even for optnone functions - clang marks every function optnone at -O0 - so
+; the index below has to end up in M0 and not in a plain SGPR.
+
+define i32 @load_i32_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-LABEL: load_i32_optnone:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s1, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ ret i32 %x
+}
+
+define void @store_i32_optnone(ptr addrspace(13) inreg %p, i32 %v) noinline optnone {
+; GFX12-LABEL: store_i32_optnone:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s1, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) %p
+ ret void
+}
>From eff2aaf71dc99788347c53d6e8ace39d568a74e9 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 00:08:17 +0300
Subject: [PATCH 07/12] Stop miscompiling VGPR-memory accesses on subtargets
without movrel
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 4 +
.../AMDGPU/AMDGPUInstructionSelector.cpp | 13 +--
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 50 +++++++--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 21 ++--
llvm/lib/Target/AMDGPU/SIInstructions.td | 7 +-
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 101 ++++++++++++++++++
6 files changed, 165 insertions(+), 31 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index f2ddf2b5d9bb1..b8004ce9925ed 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -29,6 +29,10 @@ using namespace llvm;
static bool assignIdxToM0(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+
+ // Only movrel takes its index from M0. Subtargets without it index with the
+ // VGPR indexing mode instead, which AMDGPULowerVGPREncoding enables around
+ // the move with s_set_gpr_idx_on, reading the index straight out of its SGPR.
if (!ST.hasMovrel())
return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 864a9ff82211d..43926be7efc65 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4605,14 +4605,11 @@ bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
if (!selectImpl(I, *CoverageInfo))
return false;
- // On a movrel subtarget the selected V_LOAD_IDX / V_STORE_IDX expands to an
- // M0-relative move (see AMDGPUAssignIdxToM0 and AMDGPULowerVGPREncoding). Add
- // an implicit-def of $m0: it records that the eventual move clobbers M0, and
- // - because an instruction defining a physical register is not hoisted/sunk -
- // keeps a divergent access pinned inside its waterfall loop.
- // AMDGPUAssignIdxToM0 removes it when it writes M0 for real.
- if (!Subtarget->hasMovrel())
- return true;
+ // The selected V_LOAD_IDX / V_STORE_IDX expands to an M0-relative move (see
+ // AMDGPULowerVGPREncoding), which clobbers M0 whether it indexes with movrel
+ // or with the VGPR indexing mode. Add an implicit-def of $m0 to record that,
+ // and - because an instruction defining a physical register is not
+ // hoisted/sunk - to keep a divergent access pinned inside its waterfall loop.
auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
if (LdStIdx->getIdxOp().isReg())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 90b29cafafe35..36144ed9ed767 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -48,6 +48,7 @@
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/bit.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
@@ -180,10 +181,11 @@ class AMDGPULowerVGPREncoding {
bool runOnMachineInstr(MachineInstr &MI);
/// Lower a VGPR "as memory" (address space 13) indexed load/store pseudo
- /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of M0-relative moves
- /// (v_movrels_b32 for loads, v_movreld_b32 for stores) over the wave's vector
- /// registers. M0 must already hold the dword index (see AMDGPUAssignIdxToM0).
- /// This replaces the pseudo, which is erased.
+ /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of indexed moves over
+ /// the wave's vector registers: v_movrels_b32 / v_movreld_b32 where the
+ /// subtarget has movrel, and v_mov_b32 wrapped in s_set_gpr_idx_on/off where
+ /// it indexes with the VGPR indexing mode. This replaces the pseudo, which is
+ /// erased.
void lowerLoadStoreIdx(MachineInstr &MI);
/// Compute the mode for a single \p MI given \p Ops operands
@@ -394,9 +396,9 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
const DebugLoc &DL = MI.getDebugLoc();
const bool IsStore = LdSt.mayStore();
- // $data is operand 0 of both the load (def) and store (use) pseudos; M0
- // already holds the dword index (see AMDGPUAssignIdxToM0).
+ // $data is operand 0 of both the load (def) and store (use) pseudos.
Register Data = LdSt.getDataOp().getReg();
+ MachineOperand &IdxOp = LdSt.getIdxOp();
unsigned Offset = LdSt.getOffsetOp().getImm();
unsigned NumDwords = LdSt.getBitWidth() / 32;
@@ -413,8 +415,32 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
"out of bounds VGPR 'as memory' (address space 13) access");
#endif
- unsigned Opcode =
- IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+ // Subtargets with movrel take the index from M0, which AMDGPUAssignIdxToM0
+ // has already copied it into. The rest have no movrel and index with the VGPR
+ // indexing mode instead: s_set_gpr_idx_on enables it for one operand of the
+ // moves that follow, reading the index straight out of the SGPR holding it,
+ // so no copy is needed there.
+ const bool UseGPRIdxMode = ST->useVGPRIndexMode();
+
+ MachineInstr *SetOn = nullptr;
+ if (UseGPRIdxMode) {
+ SetOn = BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
+ .add(IdxOp)
+ .addImm(IsStore ? AMDGPU::VGPRIndexMode::DST_ENABLE
+ : AMDGPU::VGPRIndexMode::SRC0_ENABLE)
+ .getInstr();
+ SetOn->getOperand(3).setIsUndef();
+ } else {
+ assert(IdxOp.isReg() && IdxOp.getReg() == AMDGPU::M0 &&
+ "movrel index should have been copied into M0");
+ }
+
+ unsigned Opcode;
+ if (UseGPRIdxMode)
+ Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
+ : AMDGPU::V_MOV_B32_indirect_read;
+ else
+ Opcode = IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
// The dword index is (M0 + $offset). Fold $offset into the base register so
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
@@ -444,6 +470,14 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
runOnMachineInstr(*Mov);
}
+ // Keep the mode switch and the moves it applies to together, so nothing is
+ // scheduled or spilled in between while indexing is enabled.
+ if (SetOn) {
+ MachineInstr *SetOff =
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
+ finalizeBundle(BB, SetOn->getIterator(), std::next(SetOff->getIterator()));
+ }
+
MI.eraseFromParent();
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b712ce06f6d5c..a9482623ce1a1 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19380,19 +19380,16 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
- // A VGPR "as memory" indexed load/store with a register index expands (on
- // movrel subtargets) to an M0-relative move. Add an implicit-def of $m0: it
- // records that the eventual move clobbers M0, and - because an instruction
- // defining a physical register is not hoisted/sunk - keeps a divergent access
- // pinned inside its waterfall loop. AMDGPUAssignIdxToM0 removes this when it
- // writes M0 for real.
+ // A VGPR "as memory" indexed load/store with a register index expands to an
+ // M0-relative move, which clobbers M0 whether it indexes with movrel or with
+ // the VGPR indexing mode. Add an implicit-def of $m0: it records that, and -
+ // because an instruction defining a physical register is not hoisted/sunk -
+ // keeps a divergent access pinned inside its waterfall loop.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
- if (getSubtarget()->hasMovrel()) {
- MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg())
- MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- }
+ MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg())
+ MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
return;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 765a5fc814d79..d404bc384f6ef 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1070,9 +1070,10 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
// a 32-bit value that may be uniform (SGPR) or divergent (VGPR); $offset is a
// constant dword offset folded in at selection.
//
-// AMDGPUAssignIdxToM0 copies $idx into M0 before register allocation, then
-// AMDGPULowerVGPREncoding lowers each into v_movrels_b32 (load) /
-// v_movreld_b32 (store) over the wave's vector registers.
+// AMDGPULowerVGPREncoding lowers each into an M0-relative move over the wave's
+// vector registers: v_movrels_b32 (load) / v_movreld_b32 (store) where the
+// subtarget has movrel, and a v_mov_b32 under the VGPR indexing mode otherwise.
+// It writes $idx to M0 there, beside the move that reads it.
// Populates the VLdStIdxOpcodeInfo searchable table (see AMDGPUMachineInstrs.h
// and AMDGPU::getVLdStIdxOpcodeInfo*), mapping each pseudo to its bit width and
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
new file mode 100644
index 0000000000000..cebc6ea23c83b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+
+; The VGPR "as memory" address space (13) on a subtarget that has no movrel.
+; gfx9 indexes with the VGPR indexing mode instead, so AMDGPULowerVGPREncoding
+; wraps the move in s_set_gpr_idx_on / s_set_gpr_idx_off, which takes the dword
+; index straight from the SGPR holding it. Nothing needs to copy the index into
+; M0, so AMDGPUAssignIdxToM0 does nothing on these subtargets.
+;
+; The mode switch and the moves it applies to are bundled, so nothing can be
+; scheduled or spilled between them while indexing is enabled.
+
+define i32 @load_i32(ptr addrspace(13) inreg %p) {
+; GFX9-LABEL: load_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %v = load i32, ptr addrspace(13) %p, align 4
+ ret i32 %v
+}
+
+define void @store_i32(ptr addrspace(13) inreg %p, i32 %v) {
+; GFX9-LABEL: store_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; More than one dword: every move has to sit inside the same mode switch.
+define i64 @load_i64(ptr addrspace(13) inreg %p) {
+; GFX9-LABEL: load_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, v1
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %v = load i64, ptr addrspace(13) %p, align 8
+ ret i64 %v
+}
+
+; A divergent pointer still needs the waterfall loop to make the index uniform
+; before the mode switch can use it.
+define i32 @load_i32_divergent(ptr addrspace(13) %p) {
+; GFX9-SDAG-LABEL: load_i32_divergent:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-SDAG-NEXT: ; %bb.2:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: load_i32_divergent:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
+; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_set_gpr_idx_off
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-GISEL-NEXT: ; %bb.2:
+; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = load i32, ptr addrspace(13) %p, align 4
+ ret i32 %v
+}
>From fdd98f89217a8b14e6d92925e1c4ebaaec8f33b5 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 02:15:42 +0300
Subject: [PATCH 08/12] Declare in TableGen that the VGPR-memory pseudos write
M0
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 11 ++------
.../AMDGPU/AMDGPUInstructionSelector.cpp | 24 ----------------
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 13 ---------
llvm/lib/Target/AMDGPU/SIInstructions.td | 13 +++++----
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 15 +++++-----
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 28 +++++++++----------
7 files changed, 32 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index b8004ce9925ed..f3972100574bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -51,14 +51,9 @@ static bool assignIdxToM0(MachineFunction &MF) {
assert(!MI.isBundled());
- // Remove the implicit-def $m0 that instruction selection added (to pin a
- // divergent access inside its waterfall loop); M0 is written for real
- // below.
- int DefIdx = MI.findRegisterDefOperandIdx(AMDGPU::M0, /*TRI=*/nullptr);
- assert(DefIdx >= 0);
- MI.removeOperand(DefIdx);
-
- // Add a copy from the index register to M0 and rewrite MI to read M0.
+ // Add a copy from the index register to M0 and rewrite MI to read M0. The
+ // pseudo goes on declaring that it writes M0: it stands for the whole
+ // sequence, and this copy is the write it describes.
// No kill flag is set on the M0 use: kill flags are deprecated and are a
// no-op on the reserved M0 register.
BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 43926be7efc65..173d8a0c2b8ef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -15,7 +15,6 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
-#include "AMDGPUMachineInstrs.h"
#include "AMDGPURegisterBankInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
@@ -4598,26 +4597,6 @@ bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
return true;
}
-bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
- // Remember where the selected machine instruction will land.
- MachineBasicBlock::iterator II = std::next(I.getIterator());
-
- if (!selectImpl(I, *CoverageInfo))
- return false;
-
- // The selected V_LOAD_IDX / V_STORE_IDX expands to an M0-relative move (see
- // AMDGPULowerVGPREncoding), which clobbers M0 whether it indexes with movrel
- // or with the VGPR indexing mode. Add an implicit-def of $m0 to record that,
- // and - because an instruction defining a physical register is not
- // hoisted/sunk - to keep a divergent access pinned inside its waterfall loop.
-
- auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
- if (LdStIdx->getIdxOp().isReg())
- LdStIdx->addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- return true;
-}
-
bool AMDGPUInstructionSelector::select(MachineInstr &I) {
if (!I.isPreISelOpcode()) {
@@ -4753,9 +4732,6 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
return selectBVHIntersectRayIntrinsic(I);
- case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE:
- return selectRegLoadStore(I);
case AMDGPU::G_SBFX:
case AMDGPU::G_UBFX:
return selectG_SBFX_UBFX(I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 7364db759ce4b..bd8ec0a769398 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -91,7 +91,6 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
bool selectCOPY_VCC_SCC(MachineInstr &I) const;
bool selectReadAnyLane(MachineInstr &I) const;
bool selectPHI(MachineInstr &I) const;
- bool selectRegLoadStore(MachineInstr &I) const;
bool selectG_TRUNC(MachineInstr &I) const;
bool selectG_SZA_EXT(MachineInstr &I) const;
bool selectG_FPEXT(MachineInstr &I) const;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a9482623ce1a1..ebeebbf9e01a3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19380,19 +19380,6 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
- // A VGPR "as memory" indexed load/store with a register index expands to an
- // M0-relative move, which clobbers M0 whether it indexes with movrel or with
- // the VGPR indexing mode. Add an implicit-def of $m0: it records that, and -
- // because an instruction defining a physical register is not hoisted/sunk -
- // keeps a divergent access pinned inside its waterfall loop.
- if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
- MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg())
- MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- return;
- }
-
if (TII->isImage(MI))
TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d404bc384f6ef..784fc5983330a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1089,10 +1089,11 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
VReg_512, VReg_1024] in {
// The units of $idx and $offset are in dwords.
//
- // VALU adds an implicit $exec use; together with the implicit-def $m0 added
- // by AdjustInstrPostInstrSelection (see hasPostISelHook), this keeps a
- // divergent-index access pinned inside its waterfall loop rather than being
- // hoisted/sunk out.
+ // The index reaches the hardware through M0, so these define it: either by
+ // writing M0 for v_movrel[sd], or through s_set_gpr_idx_on where the subtarget
+ // indexes with the VGPR indexing mode. Declaring that here also keeps a
+ // divergent-index access pinned inside its waterfall loop, since an
+ // instruction defining a physical register is not hoisted or sunk.
def V_LOAD_IDX_B#rc.Size : VPseudoInstSI <
(outs rc:$data),
(ins SReg_32:$idx, i32imm:$offset)>,
@@ -1101,7 +1102,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let hasPostISelHook = 1;
+ let Defs = [M0];
}
def V_STORE_IDX_B#rc.Size : VPseudoInstSI <
(outs),
@@ -1111,7 +1112,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let hasPostISelHook = 1;
+ let Defs = [M0];
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 28aa0ff7f0fb9..0cde03f7ab74c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -47,19 +47,20 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s1, v0
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index cebc6ea23c83b..1d3de91fd8bc2 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -60,40 +60,40 @@ define i32 @load_i32_divergent(ptr addrspace(13) %p) {
; GFX9-SDAG-LABEL: load_i32_divergent:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; GFX9-SDAG-NEXT: s_nop 1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-SDAG-NEXT: s_set_gpr_idx_off
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: load_i32_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-GISEL-NEXT: s_nop 1
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_set_gpr_idx_off
-; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-GISEL-NEXT: ; %bb.2:
+; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_set_gpr_idx_off
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
>From 336f1834cdf813f016ad7a22a52e8cc11351f2f8 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 15:26:49 +0300
Subject: [PATCH 09/12] Explain the undef operands of the VGPR-memory indexed
moves
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 14 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 224 ++++++++++++++++++
2 files changed, 237 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 36144ed9ed767..f874480a56289 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -446,6 +446,18 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
// into the addressable range so a statically out-of-bounds offset still
// resolves to a valid register.
+ //
+ // A move touches VGPR($offset + i) *plus M0*, which is only known at run
+ // time, so no operand can name the register it really reads or writes.
+ // Operands that name registers as memory rather than a value are therefore
+ // marked undef: the base of every move below, and the stored value as well
+ // when that is itself undef. Liveness of the registers behind this address
+ // space is consequently not expressed here, and correctness relies on nothing
+ // else being allocated to them - which is why frontend use of the address
+ // space is documented as discouraged.
+ const RegState DataFlags = IsStore
+ ? getUndefRegState(LdSt.getDataOp().isUndef())
+ : RegState::NoFlags;
for (unsigned i = 0; i < NumDwords; ++i) {
Register Base = AMDGPU::VGPR0 + ((Offset + i) & (NumAddressableVGPRs - 1));
Register Sub = Data;
@@ -456,7 +468,7 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
if (IsStore)
Mov = BuildMI(BB, MI, DL, TII->get(Opcode))
.addReg(Base, RegState::Undef)
- .addReg(Sub)
+ .addReg(Sub, DataFlags)
.getInstr();
else
Mov = BuildMI(BB, MI, DL, TII->get(Opcode), Sub)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
index 48d90ecaf2eff..34a8d3c2ceb9e 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -282,6 +282,230 @@ define void @copy_v8i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in
ret void
}
+define void @copy_v9i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v9i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b32 v9, v8, s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v9i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b32 v9, v8, s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <9 x i32>, ptr addrspace(13) %in
+ store <9 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v10i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v10i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v10, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v10, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v10, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b64 v10, v[8:9], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v10i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v10, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v10, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v10, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b64 v10, v[8:9], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <10 x i32>, ptr addrspace(13) %in
+ store <10 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v11i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v11i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v11, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v11, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b96 v11, v[8:10], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v11i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v11, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v11, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v11, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b96 v11, v[8:10], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <11 x i32>, ptr addrspace(13) %in
+ store <11 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v12i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v12i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v12, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v12i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <12 x i32>, ptr addrspace(13) %in
+ store <12 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
define void @copy_v16i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
; GFX12-SDAG-LABEL: copy_v16i32:
; GFX12-SDAG: ; %bb.0:
>From 3651ec8551fabd25dfbbfc49001fd506acd67f34 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 17:25:11 +0300
Subject: [PATCH 10/12] Give the VGPR-memory moves their own opcodes
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 3 ++-
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 7 ++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 24 ++++++++-----------
llvm/lib/Target/AMDGPU/SIInstructions.td | 15 ++++++++++++
4 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index f874480a56289..0fa952a89cfc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -440,7 +440,8 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
: AMDGPU::V_MOV_B32_indirect_read;
else
- Opcode = IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+ Opcode =
+ IsStore ? AMDGPU::V_MOVRELD_B32_as_mem : AMDGPU::V_MOVRELS_B32_as_mem;
// The dword index is (M0 + $offset). Fold $offset into the base register so
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 3f0fb64be4886..b109d0f5b2389 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -249,6 +249,13 @@ void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
Opcode == AMDGPU::V_FMA_MIX_BF16_t16) {
lowerT16FmaMixFP16(MI, OutMI);
return;
+ } else if (Opcode == AMDGPU::V_MOVRELS_B32_as_mem) {
+ // Indexed accesses of the VGPR "as memory" address space use their own
+ // opcodes because they index the register file rather than a tuple; they
+ // encode as the movrel they are named after.
+ Opcode = AMDGPU::V_MOVRELS_B32_e32;
+ } else if (Opcode == AMDGPU::V_MOVRELD_B32_as_mem) {
+ Opcode = AMDGPU::V_MOVRELD_B32_e32;
}
int MCOpcode = TII->pseudoToMCOpcode(Opcode);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f577f31911de4..1d9a86f7fa4a4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5681,8 +5681,6 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
- const bool IsPreRA = !MI.getMF()->getProperties().hasNoVRegs();
-
const unsigned StaticNumOps =
Desc.getNumOperands() + Desc.implicit_uses().size();
const unsigned NumImplicitOps = IsDst ? 2 : 1;
@@ -5691,7 +5689,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// post RA scheduler where the main implicit operand is killed and
// implicit-defs are added for sub-registers that remain live after this
// instruction.
- if (IsPreRA && MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
+ if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
ErrInfo = "missing implicit register operands";
return false;
}
@@ -5704,22 +5702,20 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
unsigned UseOpIdx;
- if (IsPreRA && (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
- UseOpIdx != StaticNumOps + 1)) {
+ if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
+ UseOpIdx != StaticNumOps + 1) {
ErrInfo = "movrel implicit operands should be tied";
return false;
}
}
- if (IsPreRA) {
- const MachineOperand &Src0 = MI.getOperand(Src0Idx);
- const MachineOperand &ImpUse =
- MI.getOperand(StaticNumOps + NumImplicitOps - 1);
- if (!ImpUse.isReg() || !ImpUse.isUse() ||
- !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
- ErrInfo = "src0 should be subreg of implicit vector use";
- return false;
- }
+ const MachineOperand &Src0 = MI.getOperand(Src0Idx);
+ const MachineOperand &ImpUse =
+ MI.getOperand(StaticNumOps + NumImplicitOps - 1);
+ if (!ImpUse.isReg() || !ImpUse.isUse() ||
+ !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
+ ErrInfo = "src0 should be subreg of implicit vector use";
+ return false;
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 784fc5983330a..f62e77e6c740b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1116,6 +1116,21 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
}
}
+// Copies of v_movrel[sd]_b32 for the moves the pseudos above expand into.
+//
+// The storage a movrel indexes is normally a register tuple, and the machine
+// verifier requires an implicit use of that tuple naming the register the
+// instruction really touches. Here the storage is the whole register file,
+// which no operand can name, so these carry their own opcodes and leave those
+// rules to the tuple form. AMDGPUMCInstLower maps them back for encoding.
+let VALU = 1, VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
+ Size = V_MOV_B32_e32.Size in {
+ def V_MOVRELS_B32_as_mem
+ : VPseudoInstSI<(outs VGPR_32:$vdst), (ins VRegSrc_32:$src0)>;
+ def V_MOVRELD_B32_as_mem
+ : VPseudoInstSI<(outs), (ins VGPR_32:$vdst, VSrc_b32:$src0)>;
+}
+
// Select the REG_LOAD/REG_STORE target nodes into the sized indexed pseudos.
// The pointer is a byte offset into the file; SIreg_load/SIreg_store carry the
// dword index (ptr >> 2), and an (add idx, imm) shape folds a constant dword
>From b23ff02191263c0408de2f823d80a0be422a4a1e Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:58:56 +0300
Subject: [PATCH 11/12] Pin VGPR-memory indexed accesses to EXEC and mark them
divergent
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 10 +++----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++++++++
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 15 +++++-----
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 28 +++++++++----------
4 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2e666845e5637..064a663cdf2e4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1675,7 +1675,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
// so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
// than failing to legalize.
- Actions.customIf([=](const LegalityQuery &Query) -> bool {
+ Actions.customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3510,9 +3510,9 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
- const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
- auto Two = B.buildConstant(I32, 2);
- const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+ const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ MachineInstrBuilder Two = B.buildConstant(I32, 2);
+ const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
// Normalize the value to i32 / <N x i32> so a selection pattern always
// exists (e.g. for v4i8).
@@ -3532,7 +3532,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
.addMemOperand(&MMO);
} else {
- const auto Result =
+ const MachineInstrBuilder Result =
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
.addMemOperand(&MMO);
B.buildBitcast(ValReg, Result);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 1d9a86f7fa4a4..6cb5b8b675331 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -189,6 +189,13 @@ bool SIInstrInfo::isReMaterializableImpl(
bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
assert(isVALU(MI, /*AllowLDSDMA=*/true));
+ // A VGPR "as memory" indexed access reads or writes the per-lane vector
+ // registers of the active lanes, so which lanes are active is part of what it
+ // does. Its implicit use of EXEC must not be treated as ignorable, or the
+ // access could be moved across a write to EXEC.
+ if (isa<AMDGPUMI::VLoadStoreIdxInst>(MI))
+ return true;
+
// If it is convergent it depends on EXEC.
if (MI.isConvergent())
return true;
@@ -11126,6 +11133,12 @@ ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
return ValueUniformity::Default;
}
+ // As above for the generic opcodes, but after instruction selection: an
+ // indexed load reads the wave's per-lane view of its vector registers, so
+ // even a uniform index yields a divergent value.
+ if (isa<AMDGPUMI::VLoadIdxInst>(MI))
+ return ValueUniformity::NeverUniform;
+
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 0cde03f7ab74c..28aa0ff7f0fb9 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -47,20 +47,19 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s1, v0
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-GISEL-NEXT: ; %bb.2:
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index 1d3de91fd8bc2..cebc6ea23c83b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -60,40 +60,40 @@ define i32 @load_i32_divergent(ptr addrspace(13) %p) {
; GFX9-SDAG-LABEL: load_i32_divergent:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-SDAG-NEXT: s_nop 1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-SDAG-NEXT: s_set_gpr_idx_off
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: load_i32_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-GISEL-NEXT: s_nop 1
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9-GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
-; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
-; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
; GFX9-GISEL-NEXT: s_set_gpr_idx_off
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
>From e3d5041f40c5ed5a9cac1a149ff8b18ba3ce00a3 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 3 Aug 2026 19:49:32 +0300
Subject: [PATCH 12/12] Address review: subarch triples, required pass mixin,
legalizer predicates, redundant VALU
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 ++-
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 5 +++--
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 21 +++++++++----------
llvm/lib/Target/AMDGPU/SIInstructions.td | 4 +---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll | 4 ++--
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 8 +++----
.../as-vgpr-index-demanded-bits.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-inttoptr.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-optnone.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 4 ++--
12 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index ad9cc6aa12c1d..59fb3676c9e1a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -440,7 +440,8 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
-class AMDGPUAssignIdxToM0Pass : public PassInfoMixin<AMDGPUAssignIdxToM0Pass> {
+class AMDGPUAssignIdxToM0Pass
+ : public RequiredPassInfoMixin<AMDGPUAssignIdxToM0Pass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index f3972100574bf..2c651a2d70727 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -45,9 +45,10 @@ static bool assignIdxToM0(MachineFunction &MF) {
if (!LdSt)
continue;
+ // The operand class of the index is a register class, so it never holds
+ // an immediate that would have to be moved into M0 separately.
MachineOperand &IdxOp = LdSt->getIdxOp();
- if (!IdxOp.isReg())
- continue;
+ assert(IdxOp.isReg() && "VGPR-memory index must be a register");
assert(!MI.isBundled());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 064a663cdf2e4..d7ce6a2af8c1a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -455,6 +455,11 @@ static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
return false;
+ // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
+ // to G_AMDGPU_REG_LOAD/STORE.
+ if (AS == AMDGPUAS::VGPR)
+ return false;
+
// Do not handle extending vector loads.
if (Ty.isVector() && MemSize != RegSize)
return false;
@@ -547,10 +552,6 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
- // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
- // to G_AMDGPU_REG_LOAD/STORE.
- if (Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR)
- return false;
return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
@@ -719,6 +720,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
+ const LLT VGPRPtr = GetAddrSpacePtr(AMDGPUAS::VGPR);
const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER);
const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE);
const LLT BufferStridedPtr =
@@ -1667,17 +1669,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
// 64-bits.
//
- // TODO: Should generalize bitcast action into coerce, which will also cover
- // inserting addrspacecasts.
- Actions.customIf(typeIs(1, Constant32Ptr));
-
// VGPR ("as memory") accesses are custom-lowered to the legal
// G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
// so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
// than failing to legalize.
- Actions.customIf([](const LegalityQuery &Query) -> bool {
- return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
- });
+ //
+ // TODO: Should generalize bitcast action into coerce, which will also cover
+ // inserting addrspacecasts.
+ Actions.customIf(typeInSet(1, {Constant32Ptr, VGPRPtr}));
// Turn any illegal element vectors into something easier to deal
// with. These will ultimately produce 32-bit scalar shifts to extract the
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f62e77e6c740b..aa376df4b0698 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1099,7 +1099,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
(ins SReg_32:$idx, i32imm:$offset)>,
VLdStIdxOpcodeInfo<rc.Size, 0> {
let mayLoad = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1109,7 +1108,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
(ins rc:$data, SReg_32:$idx, i32imm:$offset)>,
VLdStIdxOpcodeInfo<rc.Size, 1> {
let mayStore = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1123,7 +1121,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
// instruction really touches. Here the storage is the whole register file,
// which no operand can name, so these carry their own opcodes and leave those
// rules to the tuple form. AMDGPUMCInstLower maps them back for encoding.
-let VALU = 1, VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
+let VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
Size = V_MOV_B32_e32.Size in {
def V_MOVRELS_B32_as_mem
: VPseudoInstSI<(outs VGPR_32:$vdst), (ins VRegSrc_32:$src0)>;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
index 08d3095de3561..5ff034e4d1893 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; End-to-end lowering of the VGPR "as memory" address space (13) on a
; movrel-capable subtarget (gfx12). A load/store of a uniform (SGPR) pointer
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
index 34a8d3c2ceb9e..d4c916671796c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Copy from VGPR "as memory" (address space 13) to global memory across the
; range of legal whole-dword access sizes (32 up to 1024 bits). Each uniform
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 28aa0ff7f0fb9..4dd453b87bbf2 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; A divergent (per-lane) dword index into VGPR "as memory" (address space 13)
; is handled with a waterfall loop: for each unique index across the wave, set
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index cebc6ea23c83b..ffb673f547dd4 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.0a-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a-- -filetype=null %s
; The VGPR "as memory" address space (13) on a subtarget that has no movrel.
; gfx9 indexes with the VGPR indexing mode instead, so AMDGPULowerVGPREncoding
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 1a2689f23a1c4..e1dfb50df7fb9 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
index 393928d359f96..ae759fa010e7a 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; A constant VGPR "as memory" (address space 13) pointer formed with inttoptr:
; the dword index (256 >> 2 = 64) is a compile-time constant, so M0 is set from
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
index cd48756741766..5d25e4522447b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12
; AMDGPUAssignIdxToM0 is required lowering rather than an optimization: the
; v_movrel that AMDGPULowerVGPREncoding emits reads the dword index from M0, so
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 40263e1c975ad..75a744f49f10f 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,5 +1,5 @@
-; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
-; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
; not yet implemented. They must be rejected with a clean diagnostic on both
More information about the llvm-branch-commits
mailing list