[llvm-branch-commits] [llvm] [AMDGPU] Lower loads and stores for address space 13 (PR #209541)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 1 08:23:45 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/209541
>From 11d3dd2c756525733fb213bfede56d97c4a37bd0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 14 Jul 2026 00:06:53 -0500
Subject: [PATCH 01/19] Lower loads and stores for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 9 +
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 108 ++++
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 3 +
.../AMDGPU/AMDGPUInstructionSelector.cpp | 27 +
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 100 ++++
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 100 +++-
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 55 +++
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 71 +++
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 7 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 15 +
.../Target/AMDGPU/AMDGPUSearchableTables.td | 17 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 +
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 7 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 2 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 117 +++++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 3 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 73 ++-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 11 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 100 ++++
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 11 +
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 13 +
.../AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll | 384 +++++++++++++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 461 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 130 +++++
.../AddressSpaceVGPR/as-vgpr-inttoptr.ll | 48 ++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 31 ++
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
30 files changed, 1898 insertions(+), 24 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 2036ec03e25d6..d50ea0b22a6a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -229,6 +229,9 @@ extern char &AMDGPURegBankLegalizeID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
+void initializeAMDGPUAssignIdxToM0LegacyPass(PassRegistry &);
+extern char &AMDGPUAssignIdxToM0ID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -443,6 +446,12 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
+class AMDGPUAssignIdxToM0Pass : public PassInfoMixin<AMDGPUAssignIdxToM0Pass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
new file mode 100644
index 0000000000000..8c5de37eb46e5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -0,0 +1,108 @@
+//===- AMDGPUAssignIdxToM0.cpp - Copy VGPR-memory indices to M0 ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Copy the register index of a VGPR "as memory" (address space 13)
+/// V_LOAD_IDX / V_STORE_IDX pseudo into M0, which V_MOVREL[SD] reads when the
+/// pseudo is lowered (see AMDGPULowerVGPREncoding). This runs before register
+/// allocation so the copy to M0 is inserted while the index is still virtual.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-assign-idx-to-m0"
+
+static bool assignIdxToM0(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.hasMovrel())
+ return false;
+
+ const SIInstrInfo *TII = ST.getInstrInfo();
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
+ if (!LdSt)
+ continue;
+
+ MachineOperand &IdxOp = LdSt->getIdxOp();
+ if (!IdxOp.isReg())
+ continue;
+
+ assert(!MI.isBundled());
+
+ // Remove the implicit-def $m0 that instruction selection added (to pin a
+ // divergent access inside its waterfall loop); M0 is written for real
+ // below.
+ int DefIdx = MI.findRegisterDefOperandIdx(AMDGPU::M0, /*TRI=*/nullptr);
+ assert(DefIdx >= 0);
+ MI.removeOperand(DefIdx);
+
+ // Add a copy from the index register to M0 and rewrite MI to read M0.
+ BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
+ .add(IdxOp);
+ IdxOp.setReg(AMDGPU::M0);
+ IdxOp.setIsKill();
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
+
+namespace {
+
+class AMDGPUAssignIdxToM0Legacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUAssignIdxToM0Legacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return assignIdxToM0(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Assign Idx To M0"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPUAssignIdxToM0Pass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!assignIdxToM0(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPUAssignIdxToM0Legacy::ID = 0;
+
+char &llvm::AMDGPUAssignIdxToM0ID = AMDGPUAssignIdxToM0Legacy::ID;
+
+INITIALIZE_PASS(AMDGPUAssignIdxToM0Legacy, DEBUG_TYPE,
+ "AMDGPU Assign Idx To M0", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index d970126c726d2..dee02bbdeac34 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -290,6 +290,9 @@ def : GINodeEquiv<G_AMDGPU_FMINIMUM3, AMDGPUfminimum3>;
def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_USHORT, SIbuffer_load_ushort>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 893c7cce1a0f0..4646d48e3e8db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPURegisterBankInfo.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -4495,6 +4496,29 @@ bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
return true;
}
+bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
+ // Remember where the selected machine instruction will land.
+ MachineBasicBlock::iterator II = std::next(I.getIterator());
+
+ if (!selectImpl(I, *CoverageInfo))
+ return false;
+
+ // On a movrel subtarget the selected V_LOAD_IDX / V_STORE_IDX expands to an
+ // M0-relative move (see AMDGPUAssignIdxToM0 and AMDGPULowerVGPREncoding). Add
+ // an implicit-def of $m0: it records that the eventual move clobbers M0, and
+ // - because an instruction defining a physical register is not hoisted/sunk -
+ // keeps a divergent access pinned inside its waterfall loop.
+ // AMDGPUAssignIdxToM0 removes it when it writes M0 for real.
+ if (!Subtarget->hasMovrel())
+ return true;
+
+ auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
+ if (LdStIdx->getIdxOp().isReg())
+ LdStIdx->addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
+ return true;
+}
+
bool AMDGPUInstructionSelector::select(MachineInstr &I) {
if (!I.isPreISelOpcode()) {
@@ -4630,6 +4654,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
return selectBVHIntersectRayIntrinsic(I);
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ return selectRegLoadStore(I);
case AMDGPU::G_SBFX:
case AMDGPU::G_UBFX:
return selectG_SBFX_UBFX(I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index ce3a090651d0e..98f0cf2c3d009 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -91,6 +91,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
bool selectCOPY_VCC_SCC(MachineInstr &I) const;
bool selectReadAnyLane(MachineInstr &I) const;
bool selectPHI(MachineInstr &I) const;
+ bool selectRegLoadStore(MachineInstr &I) const;
bool selectG_TRUNC(MachineInstr &I) const;
bool selectG_SZA_EXT(MachineInstr &I) const;
bool selectG_FPEXT(MachineInstr &I) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 53e579fd06e26..a99a4c67afcb3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -16,6 +16,7 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUTargetMachine.h"
#include "SIInstrInfo.h"
@@ -431,6 +432,8 @@ static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
// register bank/uniformity and if the memory is invariant or not written in a
// kernel.
return IsLoad ? 512 : 128;
+ case AMDGPUAS::VGPR:
+ return 1024;
default:
// FIXME: Flat addresses may contextually need to be split to 32-bit parts
// if they may alias scratch depending on the subtarget. This needs to be
@@ -547,10 +550,23 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
+ // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
+ // to G_AMDGPU_REG_LOAD/STORE.
+ if (Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR)
+ return false;
return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
+// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
+// memory access producing/consuming a ValSize-bit value. Only whole-dword
+// accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported
+// for now; sub-dword (8/16-bit) support lands later.
+static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+ return MemSize == ValSize &&
+ AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+}
+
/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
@@ -1663,6 +1679,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// inserting addrspacecasts.
Actions.customIf(typeIs(1, Constant32Ptr));
+ // VGPR ("as memory") accesses are custom-lowered to the legal
+ // G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
+ // so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
+ // than failing to legalize.
+ Actions.customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
+ });
+
// Turn any illegal element vectors into something easier to deal
// with. These will ultimately produce 32-bit scalar shifts to extract the
// parts anyway.
@@ -3450,6 +3474,75 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
+/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into a legal
+/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE indexed by the pointer's dword offset
+/// (pointer >> 2). Parallels the SelectionDAG LowerLoadStoreVGPR.
+static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *B.getMRI();
+ MachineMemOperand &MMO = **MI.memoperands_begin();
+
+ const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ Register ValReg = MI.getOperand(0).getReg();
+ Register PtrReg = MI.getOperand(1).getReg();
+
+ const LLT ValTy = MRI.getType(ValReg);
+ const unsigned ValSize = ValTy.getSizeInBits();
+ // The GISel selection patterns for the indexed pseudos - and for the shift /
+ // readfirstlane that compute the index - match the extended integer LLT, so
+ // build the dword index (and the normalized register value below) with
+ // integer types rather than plain scalars.
+ const LLT I32 = LLT::integer(32);
+
+ // Only whole-dword, non-extending/non-truncating accesses are implemented.
+ // Reject anything else with a diagnostic instead of failing to legalize
+ // (sub-dword support lands in a later change).
+ if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
+ ValSize)) {
+ const Function &F = B.getMF().getFunction();
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F,
+ "unsupported access of VGPR 'as memory' address space (13); only "
+ "whole-dword loads and stores are implemented",
+ MI.getDebugLoc()));
+ if (!IsStore)
+ B.buildUndef(ValReg);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ // Normalize the value to i32 / <N x i32> so a selection pattern always
+ // exists (e.g. for v4i8).
+ LLT RegTy = ValTy;
+ if (ValTy.getScalarSizeInBits() != 32) {
+ unsigned NumDwords = ValSize / 32;
+ RegTy = NumDwords == 1 ? I32 : LLT::fixed_vector(NumDwords, I32);
+ }
+
+ if (IsStore) {
+ Register Value = ValReg;
+ if (RegTy != ValTy)
+ Value = B.buildBitcast(RegTy, Value).getReg(0);
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE, {}, {Value, Index.getReg(0)})
+ .addMemOperand(&MMO);
+ } else if (RegTy == ValTy) {
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
+ .addMemOperand(&MMO);
+ } else {
+ const auto Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
+ .addMemOperand(&MMO);
+ B.buildBitcast(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+}
+
bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
@@ -3460,6 +3553,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
+ if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ return lowerLoadStoreVGPR(Helper, MI);
+
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
@@ -3548,6 +3644,10 @@ bool AMDGPULegalizerInfo::legalizeStore(LegalizerHelper &Helper,
Register DataReg = MI.getOperand(0).getReg();
LLT DataTy = MRI.getType(DataReg);
+ if (MRI.getType(MI.getOperand(1).getReg()).getAddressSpace() ==
+ AMDGPUAS::VGPR)
+ return lowerLoadStoreVGPR(Helper, MI);
+
if (hasBufferRsrcWorkaround(DataTy)) {
Observer.changingInstr(MI);
castBufferRsrcArgToV4I32(MI, B, 0);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 75c3dd3b1de09..b7e26e2376354 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -42,9 +42,11 @@
#include "AMDGPULowerVGPREncoding.h"
#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
#include "GCNSubtarget.h"
#include "SIDefines.h"
#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/bit.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/Support/Debug.h"
@@ -132,6 +134,7 @@ class AMDGPULowerVGPREncoding {
bool run(MachineFunction &MF);
private:
+ const GCNSubtarget *ST;
const SIInstrInfo *TII;
const SIRegisterInfo *TRI;
@@ -177,6 +180,13 @@ class AMDGPULowerVGPREncoding {
/// Handle single \p MI. \return true if changed.
bool runOnMachineInstr(MachineInstr &MI);
+ /// Lower a VGPR "as memory" (address space 13) indexed load/store pseudo
+ /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of M0-relative moves
+ /// (v_movrels_b32 for loads, v_movreld_b32 for stores) over the wave's vector
+ /// registers. M0 must already hold the dword index (see AMDGPUAssignIdxToM0).
+ /// This replaces the pseudo, which is erased.
+ void lowerLoadStoreIdx(MachineInstr &MI);
+
/// Compute the mode for a single \p MI given \p Ops operands
/// bit mapping. Optionally takes second array \p Ops2 for VOPD.
/// If provided and an operand from \p Ops is not a VGPR, then \p Ops2
@@ -379,6 +389,63 @@ bool AMDGPULowerVGPREncoding::runOnMachineInstr(MachineInstr &MI) {
return false;
}
+void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
+ auto &LdSt = cast<AMDGPUMI::VLoadStoreIdxInst>(MI);
+ MachineBasicBlock &BB = *MI.getParent();
+ const DebugLoc &DL = MI.getDebugLoc();
+ const bool IsStore = LdSt.mayStore();
+
+ // $data is operand 0 of both the load (def) and store (use) pseudos; M0
+ // already holds the dword index (see AMDGPUAssignIdxToM0).
+ Register Data = LdSt.getDataOp().getReg();
+ unsigned Offset = LdSt.getOffsetOp().getImm();
+ unsigned NumDwords = LdSt.getBitWidth() / 32;
+
+ // A statically out-of-range dword offset would fold into a base VGPR outside
+ // the addressable register file - an out-of-bounds access of the VGPR
+ // "as memory" (address space 13) region. When the whole file is addressable
+ // the index is allowed to wrap; otherwise it must stay in range.
+#ifndef NDEBUG
+ unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
+ MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
+ bool AllowOffsetWrap =
+ NumAddressableVGPRs == AMDGPU::IsaInfo::getTotalNumVGPRs(*ST);
+ assert((AllowOffsetWrap || Offset + NumDwords <= NumAddressableVGPRs) &&
+ "out of bounds VGPR 'as memory' (address space 13) access");
+#endif
+
+ unsigned Opcode =
+ IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+
+ // The dword index is (M0 + $offset). Fold $offset into the base register so
+ // each dword i reads/writes VGPR($offset + i) relative to M0.
+ for (unsigned i = 0; i < NumDwords; ++i) {
+ Register Base = AMDGPU::VGPR0 + Offset + i;
+ Register Sub = Data;
+ if (NumDwords != 1)
+ Sub = TRI->getSubReg(Data, TRI->getSubRegFromChannel(i));
+
+ MachineInstr *Mov;
+ if (IsStore)
+ Mov = BuildMI(BB, MI, DL, TII->get(Opcode))
+ .addReg(Base, RegState::Undef)
+ .addReg(Sub)
+ .getInstr();
+ else
+ Mov = BuildMI(BB, MI, DL, TII->get(Opcode), Sub)
+ .addReg(Base, RegState::Undef)
+ .getInstr();
+
+ // On subtargets with more than 256 addressable VGPRs the referenced
+ // register may need high address bits; reuse the S_SET_VGPR_MSB machinery
+ // to encode them. This is a no-op on movrel-only (<=256 VGPR) subtargets.
+ if (ST->has1024AddressableVGPRs())
+ runOnMachineInstr(*Mov);
+ }
+
+ MI.eraseFromParent();
+}
+
MachineBasicBlock::instr_iterator
AMDGPULowerVGPREncoding::handleClause(MachineBasicBlock::instr_iterator I) {
if (!ClauseRemaining)
@@ -570,12 +637,14 @@ bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
}
bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!ST.has1024AddressableVGPRs())
- return false;
+ ST = &MF.getSubtarget<GCNSubtarget>();
+ TII = ST->getInstrInfo();
+ TRI = ST->getRegisterInfo();
- TII = ST.getInstrInfo();
- TRI = ST.getRegisterInfo();
+ // The S_SET_VGPR_MSB encoding is only required on subtargets with more than
+ // 256 addressable VGPRs (gfx1250). On movrel-only subtargets the pass still
+ // runs, but only to lower the VGPR "as memory" indexed load/store pseudos.
+ const bool LowerVGPRMSBs = ST->has1024AddressableVGPRs();
LLVM_DEBUG(dbgs() << "*** AMDGPULowerVGPREncoding on " << MF.getName()
<< " ***\n");
@@ -592,6 +661,19 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
<< ":\n");
for (auto &MI : llvm::make_early_inc_range(MBB.instrs())) {
+ // Lower VGPR "as memory" indexed load/store pseudos on any subtarget that
+ // reaches this pass (movrel-only or gfx1250). This replaces the pseudo.
+ if (isa<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ lowerLoadStoreIdx(MI);
+ Changed = true;
+ continue;
+ }
+
+ // The remaining work only inserts VGPR MSB encoding, which is unnecessary
+ // on movrel-only subtargets.
+ if (!LowerVGPRMSBs)
+ continue;
+
if (MI.isMetaInstruction())
continue;
@@ -624,7 +706,7 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
}
if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
- ST.hasSetregVGPRMSBFixup()) {
+ ST->hasSetregVGPRMSBFixup()) {
Changed |= handleSetregMode(MI);
continue;
}
@@ -648,8 +730,10 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
}
// Reset the mode if we are falling through.
- LLVM_DEBUG(dbgs() << " end of BB, resetting mode\n");
- resetMode(MBB.instr_end());
+ if (LowerVGPRMSBs) {
+ LLVM_DEBUG(dbgs() << " end of BB, resetting mode\n");
+ resetMode(MBB.instr_end());
+ }
}
return Changed;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
new file mode 100644
index 0000000000000..697c63e2079d1
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -0,0 +1,55 @@
+//===-- AMDGPUMachineInstrs.cpp -*- C++ -*---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Convenience wrappers and helpers for AMDGPU-specific machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUMachineInstrs.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+using namespace AMDGPUMI;
+
+unsigned VLoadStoreIdxInst::getBitWidth() const {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ if (!Info)
+ llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
+ return Info->BitWidth;
+}
+
+int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
+ if (!Info)
+ return -1;
+ return Info->Opcode;
+}
+
+unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
+ int Opcode = tryGetOpcodeForBitWidth(Bits);
+ assert(Opcode != -1);
+ return Opcode;
+}
+
+int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
+ if (!Info)
+ return -1;
+ return Info->Opcode;
+}
+
+unsigned VStoreIdxInst::getOpcodeForBitWidth(unsigned Bits) {
+ int Opcode = tryGetOpcodeForBitWidth(Bits);
+ assert(Opcode != -1);
+ return Opcode;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
new file mode 100644
index 0000000000000..24e24aab53152
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -0,0 +1,71 @@
+//===-- AMDGPUMachineInstrs.h -*- C++ -*-----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Convenience wrappers and helpers for AMDGPU-specific machine instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
+
+#include "SIInstrInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+
+namespace llvm {
+namespace AMDGPUMI {
+
+// Wrapper for the whole-dword VGPR "as memory" (address space 13) indexed
+// load/store pseudos (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>). Operand layout:
+// load: (outs data), (ins idx, offset)
+// store: (outs), (ins data, idx, offset)
+// so data/idx/offset are always operands 0/1/2.
+class VLoadStoreIdxInst : public MachineInstr {
+public:
+ MachineOperand &getDataOp() { return getOperand(0); }
+ MachineOperand &getIdxOp() { return getOperand(1); }
+ MachineOperand &getOffsetOp() { return getOperand(2); }
+ const MachineOperand &getDataOp() const { return getOperand(0); }
+ const MachineOperand &getIdxOp() const { return getOperand(1); }
+ const MachineOperand &getOffsetOp() const { return getOperand(2); }
+
+ unsigned getBitWidth() const;
+
+ static bool classof(const MachineInstr *MI) {
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ }
+};
+
+class VLoadIdxInst : public VLoadStoreIdxInst {
+public:
+ static int tryGetOpcodeForBitWidth(unsigned Bits);
+ static unsigned getOpcodeForBitWidth(unsigned Bits);
+
+ static bool classof(const MachineInstr *MI) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ return Info && !Info->IsStore;
+ }
+};
+
+class VStoreIdxInst : public VLoadStoreIdxInst {
+public:
+ static int tryGetOpcodeForBitWidth(unsigned Bits);
+ static unsigned getOpcodeForBitWidth(unsigned Bits);
+
+ static bool classof(const MachineInstr *MI) {
+ const AMDGPU::VLdStIdxOpcodeInfo *Info =
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ return Info && Info->IsStore;
+ }
+};
+
+} // end namespace AMDGPUMI
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEINSTRS_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 6da139ea0b59c..af2b377df5eb4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -116,6 +116,7 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
#endif
MACHINE_FUNCTION_PASS("amdgpu-asm-printer", AMDGPUAsmPrinterPass())
+MACHINE_FUNCTION_PASS("amdgpu-assign-idx-to-m0", AMDGPUAssignIdxToM0Pass())
MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 0072ef76464cc..56f81a597a836 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1392,6 +1392,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
{{VgprV2S16},
{VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
+ // VGPR ("as memory") indexed load/store: the data is a VGPR value of any
+ // register class; the dword index is made uniform (waterfall) as an SGPR.
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD}).Any({{BRC}, {{VgprBRC}, {Sgpr32_WF}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 15c196c2032de..e292e4a0691a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3088,6 +3088,14 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE: {
+ // The dword index (operand 1) must be uniform; a divergent index needs a
+ // waterfall loop.
+ applyDefaultMapping(OpdMapper);
+ executeInWaterfallLoop(B, MI, {1});
+ return;
+ }
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
@@ -4487,6 +4495,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
}
break;
}
+ case AMDGPU::G_AMDGPU_REG_LOAD:
+ case AMDGPU::G_AMDGPU_REG_STORE: {
+ // data/result is a VGPR value; the dword index is uniform (SGPR).
+ OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+ OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ break;
+ }
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 81738d3fdc65a..b4a069efe1b3e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -429,3 +429,20 @@ def AMDGPUImageDMaskIntrinsicTable : GenericTable {
let PrimaryKeyName = "getAMDGPUImageDMaskIntrinsic";
let PrimaryKeyEarlyOut = 1;
}
+
+//===----------------------------------------------------------------------===//
+// V_LOAD/STORE_IDX opcode mapping table.
+//===----------------------------------------------------------------------===//
+
+def VLdStIdxOpcodeInfoTable : GenericTable {
+ let FilterClass = "VLdStIdxOpcodeInfo";
+ let CppTypeName = "VLdStIdxOpcodeInfo";
+ let Fields = ["Opcode", "BitWidth", "IsStore"];
+ let PrimaryKey = ["Opcode"];
+ let PrimaryKeyName = "getVLdStIdxOpcodeInfoByOpcodeImpl";
+}
+
+def getVLdStIdxOpcodeInfoByKeyImpl : SearchIndex {
+ let Table = VLdStIdxOpcodeInfoTable;
+ let Key = ["BitWidth", "IsStore"];
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 544245875f8b7..0bd83203446f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -693,6 +693,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizePass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
+ initializeAMDGPUAssignIdxToM0LegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1835,6 +1836,10 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Copy the VGPR "as memory" load/store index into M0 before register
+ // allocation; the movrel emitted later by AMDGPULowerVGPREncoding reads it.
+ addPass(&AMDGPUAssignIdxToM0ID);
+
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
@@ -2645,6 +2650,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
+ // Set up M0 for the movrel that expands a VGPR "as memory" indexed access.
+ // Run before allocation so the index computation coalesces into M0.
+ addMachineFunctionPass(AMDGPUAssignIdxToM0Pass(), PMW);
+
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 4e2dad5c2d253..efba9cd29c22b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1126,13 +1126,16 @@ bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
// Loads from the private and flat address spaces are divergent, because
// threads can execute the load instruction with the same inputs and get
- // different results.
+ // different results. The same is true of the VGPR ("as memory") address
+ // space: it is a per-lane view of the vector registers, so an access at a
+ // uniform offset still yields a per-lane (divergent) value.
//
// All other loads are not divergent, because if threads issue loads with the
// same arguments, they will always get the same result.
if (const LoadInst *Load = dyn_cast<LoadInst>(V))
return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
- Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
+ Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
+ Load->getPointerAddressSpace() == AMDGPUAS::VGPR;
// Atomics are divergent because they are executed sequentially: when an
// atomic operation refers to the same address in each thread, then each
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..054fc1d14d7cb 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -47,6 +47,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUArgumentUsageInfo.cpp
AMDGPUAsanInstrumentation.cpp
AMDGPUAsmPrinter.cpp
+ AMDGPUAssignIdxToM0.cpp
AMDGPUAtomicOptimizer.cpp
AMDGPUAttributor.cpp
AMDGPUBarrierLatency.cpp
@@ -83,6 +84,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPULowerExecSync.cpp
AMDGPUSwLowerLDS.cpp
AMDGPUMachineFunctionInfo.cpp
+ AMDGPUMachineInstrs.cpp
AMDGPUMachineModuleInfo.cpp
AMDGPUMacroFusion.cpp
AMDGPUMCInstLower.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a8b4d1f9a568b..48b6689769913 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUMachineInstrs.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUSelectionDAGInfo.h"
#include "AMDGPUTargetMachine.h"
@@ -13457,6 +13458,87 @@ static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
return true;
}
+// Lower a load or store of the VGPR ("as memory") address space (13) to a
+// REG_LOAD / REG_STORE target node. The 32-bit pointer is a byte offset into
+// the wave's view of its vector registers; the target node carries the dword
+// index (pointer >> 2). Recognizing a constant dword offset is left to the
+// selection patterns, which fold an (add index, imm) shape into the pseudo.
+//
+// TODO: sub-dword (8/16-bit) accesses are not yet supported; they are
+// diagnosed as unsupported below.
+SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ MemSDNode *MemOp = cast<MemSDNode>(Op);
+ EVT MemVT = MemOp->getMemoryVT();
+ unsigned BitWidth = MemVT.getSizeInBits();
+
+ // Only whole-dword, non-extending/non-truncating accesses are implemented.
+ // Reject anything else with a diagnostic (replacing the value with poison)
+ // instead of failing instruction selection. Both callers - operation
+ // legalization and the pre-ISel combine - replace the node with this result,
+ // so the diagnostic is emitted exactly once.
+ auto reportUnsupported = [&]() -> SDValue {
+ const Function &F = DAG.getMachineFunction().getFunction();
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ F,
+ "unsupported access of VGPR 'as memory' address space (13); only "
+ "whole-dword loads and stores are implemented",
+ DL.getDebugLoc()));
+ if (isa<StoreSDNode>(MemOp))
+ return MemOp->getChain();
+ return DAG.getMergeValues(
+ {DAG.getPOISON(Op.getValueType()), MemOp->getChain()}, DL);
+ };
+
+ if (BitWidth < 32)
+ return reportUnsupported();
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD)
+ return reportUnsupported();
+ if (AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(BitWidth) == -1)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore())
+ return reportUnsupported();
+ if (AMDGPUMI::VStoreIdxInst::tryGetOpcodeForBitWidth(BitWidth) == -1)
+ return reportUnsupported();
+ }
+
+ SDValue Chain = MemOp->getChain();
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, MemOp->getBasePtr(),
+ DAG.getConstant(2, DL, MVT::i32));
+
+ // View the access as i32 / <N x i32> when the memory type is not register
+ // legal (e.g. v4i8), bitcasting the value across.
+ EVT RegVT = MemVT;
+ if (!isTypeLegal(RegVT)) {
+ unsigned NumDwords = BitWidth / 32;
+ RegVT = NumDwords == 1
+ ? EVT(MVT::i32)
+ : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumDwords);
+ }
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ if (RegVT != MemVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, RegVT, Value);
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ SDValue NewLoad = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD, DL, DAG.getVTList(RegVT, MVT::Other), {Chain, Index},
+ MemVT, LoadOp->getMemOperand());
+ if (RegVT == MemVT)
+ return NewLoad;
+ SDValue Value = DAG.getNode(ISD::BITCAST, DL, MemVT, NewLoad);
+ return DAG.getMergeValues({Value, NewLoad.getValue(1)}, DL);
+}
+
SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
LoadSDNode *Load = cast<LoadSDNode>(Op);
@@ -13464,6 +13546,9 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
EVT MemVT = Load->getMemoryVT();
MachineMemOperand *MMO = Load->getMemOperand();
+ if (Load->getAddressSpace() == AMDGPUAS::VGPR)
+ return LowerLoadStoreVGPR(Op, DAG);
+
if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
if (MemVT == MVT::i16 && isTypeLegal(MVT::i16))
return SDValue();
@@ -14136,6 +14221,9 @@ SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
StoreSDNode *Store = cast<StoreSDNode>(Op);
EVT VT = Store->getMemoryVT();
+ if (Store->getAddressSpace() == AMDGPUAS::VGPR)
+ return LowerLoadStoreVGPR(Op, DAG);
+
if (VT == MVT::i1) {
return DAG.getTruncStore(
Store->getChain(), DL,
@@ -19014,6 +19102,19 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
if (auto Res = promoteUniformOpToI32(SDValue(N, 0), DCI))
return Res;
break;
+ case ISD::LOAD:
+ // Lower a VGPR ("as memory") address space (13) load to a REG_LOAD target
+ // node. Done here (not via operation legalization) so it also fires at -O0,
+ // where a scalar load is otherwise Legal and never reaches LowerLOAD.
+ if (cast<LoadSDNode>(N)->getAddressSpace() == AMDGPUAS::VGPR)
+ if (SDValue V = LowerLoadStoreVGPR(SDValue(N, 0), DCI.DAG))
+ return V;
+ break;
+ case ISD::STORE:
+ if (cast<StoreSDNode>(N)->getAddressSpace() == AMDGPUAS::VGPR)
+ if (SDValue V = LowerLoadStoreVGPR(SDValue(N, 0), DCI.DAG))
+ return V;
+ break;
default:
break;
}
@@ -19613,6 +19714,22 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
+ // A VGPR "as memory" indexed load/store with a register index expands (on
+ // movrel subtargets) to an M0-relative move. Add an implicit-def of $m0: it
+ // records that the eventual move clobbers M0, and - because an instruction
+ // defining a physical register is not hoisted/sunk - keeps a divergent access
+ // pinned inside its waterfall loop. AMDGPUAssignIdxToM0 removes this when it
+ // writes M0 for real.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (getSubtarget()->hasMovrel()) {
+ MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg())
+ MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
+ }
+ return;
+ }
+
if (TII->isImage(MI))
TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 8ece9f279a100..4aab0a9ba4d01 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -122,6 +122,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
+ // Lower a load/store of the VGPR ("as memory") address space (13) to a
+ // REG_LOAD/REG_STORE target node indexed by the pointer's dword offset.
+ SDValue LowerLoadStoreVGPR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFastUnsafeFDIV64(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3094c1390e9cf..c648da68f0f94 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
+#include "AMDGPUMachineInstrs.h"
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
@@ -619,6 +620,18 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return true;
}
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ BaseOp = &LdStIdx->getIdxOp();
+ OffsetOp = &LdStIdx->getOffsetOp();
+
+ BaseOps.push_back(BaseOp);
+ Offset = OffsetOp->getImm() * 4; // Offset has units of dwords.
+
+ // Get appropriate operand, and compute width accordingly.
+ Width = LocationSize::precise(LdStIdx->getBitWidth() / 8);
+ return true;
+ }
+
return false;
}
@@ -4202,10 +4215,20 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
return false;
- if (isLDSDMA(MIa) || isLDSDMA(MIb))
+ if (MIa.isBundle() || MIb.isBundle())
return false;
- if (MIa.isBundle() || MIb.isBundle())
+ // VGPR "as memory" indexed accesses only alias each other, and then only
+ // when their [idx+offset, idx+offset+width) dword ranges overlap.
+ const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa);
+ const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb);
+ if (IsLdStIdxA || IsLdStIdxB) {
+ if (IsLdStIdxA && IsLdStIdxB)
+ return checkInstOffsetsDoNotOverlap(MIa, MIb);
+ return true;
+ }
+
+ if (isLDSDMA(MIa) || isLDSDMA(MIb))
return false;
// TODO: Should we check the address space from the MachineMemOperand? That
@@ -5821,6 +5844,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
+ const bool IsPreRA = !MI.getMF()->getProperties().hasNoVRegs();
const unsigned StaticNumOps =
Desc.getNumOperands() + Desc.implicit_uses().size();
@@ -5830,7 +5854,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// post RA scheduler where the main implicit operand is killed and
// implicit-defs are added for sub-registers that remain live after this
// instruction.
- if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
+ if (IsPreRA && MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
ErrInfo = "missing implicit register operands";
return false;
}
@@ -5843,20 +5867,22 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
unsigned UseOpIdx;
- if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
- UseOpIdx != StaticNumOps + 1) {
+ if (IsPreRA && (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
+ UseOpIdx != StaticNumOps + 1)) {
ErrInfo = "movrel implicit operands should be tied";
return false;
}
}
- const MachineOperand &Src0 = MI.getOperand(Src0Idx);
- const MachineOperand &ImpUse
- = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
- if (!ImpUse.isReg() || !ImpUse.isUse() ||
- !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
- ErrInfo = "src0 should be subreg of implicit vector use";
- return false;
+ if (IsPreRA) {
+ const MachineOperand &Src0 = MI.getOperand(Src0Idx);
+ const MachineOperand &ImpUse =
+ MI.getOperand(StaticNumOps + NumImplicitOps - 1);
+ if (!ImpUse.isReg() || !ImpUse.isUse() ||
+ !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
+ ErrInfo = "src0 should be subreg of implicit vector use";
+ return false;
+ }
}
}
@@ -7688,6 +7714,17 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
return CreatedBB;
}
+ // A VGPR "as memory" indexed load/store needs its dword index in an SGPR (it
+ // becomes M0). A divergent (VGPR) index is made uniform with a waterfall
+ // loop that executes the access once per unique index across the wave.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ MachineOperand *Idx = &LdStIdx->getIdxOp();
+ if (Idx->isReg() && Idx->getReg().isVirtual() &&
+ !RI.isSGPRClass(MRI.getRegClass(Idx->getReg())))
+ CreatedBB = generateWaterFallLoop(*this, MI, {Idx}, MDT);
+ return CreatedBB;
+ }
+
// Legalize PHI
// The register class of the operands must be the same type as the register
// class of the output.
@@ -11169,9 +11206,16 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
return ValueUniformity::Default;
}
+ // A VGPR ("as memory") indexed load is always divergent: it reads the wave's
+ // per-lane view of its vector registers, so even a uniform index yields a
+ // per-lane (divergent) value.
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ return ValueUniformity::NeverUniform;
+
// Loads from the private and flat address spaces are divergent, because
// threads can execute the load instruction with the same inputs and get
- // different results.
+ // different results. The VGPR address space is likewise divergent (see
+ // above; this covers a G_LOAD not yet legalized to G_AMDGPU_REG_LOAD).
//
// All other loads are not divergent, because if threads issue loads with the
// same arguments, they will always get the same result.
@@ -11182,7 +11226,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {
return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
- mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
+ mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS ||
+ mmo->getAddrSpace() == AMDGPUAS::VGPR;
})) {
// At least one MMO in a non-global address space.
return ValueUniformity::NeverUniform;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 00e81b699be72..7bc293b060820 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -59,6 +59,17 @@ def GFX10Gen : GFXGen<isGFX10Only, "GFX10", "_gfx10", SIEncodingFamily.G
// modifier behavior with dx10_enable.
def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPUnaryOp>;
+// VGPR address space (13) load/store with a dword index operand. The index is
+// the byte offset into the wave's view of its vector registers, divided by 4.
+def SDTRegIdxLoad : SDTypeProfile<1, 1,
+ [SDTCisVT<1, i32>]>; // dword_index
+def SDTRegIdxStore : SDTypeProfile<0, 2,
+ [SDTCisVT<1, i32>]>; // data, dword_index
+def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index eb23b6fea26b3..968b9f24f4803 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1061,6 +1061,88 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
} // End Uses = [EXEC], Defs = [M0, EXEC]
+//===----------------------------------------------------------------------===//
+// VGPR "as memory" indexed load/store pseudos (address space 13)
+//===----------------------------------------------------------------------===//
+
+// V_LOAD_IDX_B<N> / V_STORE_IDX_B<N> load or store N bits from/to the wave's
+// view of its vector registers, at a dword index of ($idx + $offset). $idx is
+// a 32-bit value that may be uniform (SGPR) or divergent (VGPR); $offset is a
+// constant dword offset folded in at selection.
+//
+// AMDGPUAssignIdxToM0 copies $idx into M0 before register allocation, then
+// AMDGPULowerVGPREncoding lowers each into v_movrels_b32 (load) /
+// v_movreld_b32 (store) over the wave's vector registers.
+
+// Populates the VLdStIdxOpcodeInfo searchable table (see AMDGPUMachineInstrs.h
+// and AMDGPU::getVLdStIdxOpcodeInfo*), mapping each pseudo to its bit width and
+// load/store direction.
+class VLdStIdxOpcodeInfo<int size, bit isStore> {
+ Instruction Opcode = !cast<Instruction>(NAME);
+ bits<12> BitWidth = size;
+ bit IsStore = isStore;
+}
+
+foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
+ VReg_224, VReg_256, VReg_288, VReg_320, VReg_352, VReg_384,
+ VReg_512, VReg_1024] in {
+ // The units of $idx and $offset are in dwords.
+ //
+ // VALU adds an implicit $exec use; together with the implicit-def $m0 added
+ // by AdjustInstrPostInstrSelection (see hasPostISelHook), this keeps a
+ // divergent-index access pinned inside its waterfall loop rather than being
+ // hoisted/sunk out.
+ def V_LOAD_IDX_B#rc.Size : VPseudoInstSI <
+ (outs rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset)>,
+ VLdStIdxOpcodeInfo<rc.Size, 0> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let hasPostISelHook = 1;
+ }
+ def V_STORE_IDX_B#rc.Size : VPseudoInstSI <
+ (outs),
+ (ins rc:$data, SReg_32:$idx, i32imm:$offset)>,
+ VLdStIdxOpcodeInfo<rc.Size, 1> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let hasPostISelHook = 1;
+ }
+}
+
+// Select the REG_LOAD/REG_STORE target nodes into the sized indexed pseudos.
+// The pointer is a byte offset into the file; SIreg_load/SIreg_store carry the
+// dword index (ptr >> 2), and an (add idx, imm) shape folds a constant dword
+// offset into the pseudo's $offset operand.
+multiclass VRegIdxLoadStorePat<ValueType vt> {
+ defvar load_inst = !cast<Instruction>("V_LOAD_IDX_B"#vt.Size);
+ defvar store_inst = !cast<Instruction>("V_STORE_IDX_B"#vt.Size);
+
+ def : GCNPat<
+ (vt (SIreg_load (add i32:$idx, (i32 imm:$offset)))),
+ (load_inst $idx, imm:$offset)>;
+ def : GCNPat<
+ (vt (SIreg_load i32:$idx)),
+ (load_inst $idx, 0)>;
+ def : GCNPat<
+ (SIreg_store vt:$data, (add i32:$idx, (i32 imm:$offset))),
+ (store_inst $data, $idx, imm:$offset)>;
+ def : GCNPat<
+ (SIreg_store vt:$data, i32:$idx),
+ (store_inst $data, $idx, 0)>;
+}
+
+foreach vt = !listconcat(
+ Reg32Types.types, Reg64Types.types, Reg96Types.types, Reg128Types.types,
+ Reg160Types.types, Reg192Types.types, Reg224Types.types, Reg256Types.types,
+ Reg288Types.types, Reg320Types.types, Reg352Types.types, Reg384Types.types,
+ Reg512Types.types, Reg1024Types.types) in
+defm : VRegIdxLoadStorePat<vt>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4816,6 +4898,24 @@ def G_AMDGPU_BUFFER_STORE_FORMAT_D16 : BufferStoreGenericInstruction;
def G_AMDGPU_TBUFFER_STORE_FORMAT : TBufferStoreGenericInstruction;
def G_AMDGPU_TBUFFER_STORE_FORMAT_D16 : TBufferStoreGenericInstruction;
+// GlobalISel equivalents of the REG_LOAD / REG_STORE target nodes: a load or
+// store of the VGPR ("as memory") address space, indexed by the pointer's
+// dword offset. They select to the V_LOAD_IDX_B<N> / V_STORE_IDX_B<N> pseudos
+// through the same TableGen patterns (see GINodeEquiv in AMDGPUGISel.td).
+def G_AMDGPU_REG_LOAD : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index a0648da67c06d..df114c2f4f73e 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -478,9 +478,20 @@ struct FP4FP8DstByteSelInfo {
#define GET_getMFMA_F8F6F4_WithSize_IMPL
#define GET_isMFMA_F8F6F4Table_IMPL
#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
+#define GET_VLdStIdxOpcodeInfoTable_DECL
+#define GET_VLdStIdxOpcodeInfoTable_IMPL
#include "AMDGPUGenSearchableTables.inc"
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByOpcode(unsigned Opc) {
+ return getVLdStIdxOpcodeInfoByOpcodeImpl(Opc);
+}
+
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByKey(uint16_t BitWidth,
+ bool IsStore) {
+ return getVLdStIdxOpcodeInfoByKeyImpl(BitWidth, IsStore);
+}
+
int getMTBUFBaseOpcode(unsigned Opc) {
const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
return Info ? Info->BaseOpcode : -1;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index d429584b00999..d1c735efd50a4 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -487,6 +487,19 @@ struct MIMGInfo {
LLVM_READONLY
const MIMGInfo *getMIMGInfo(unsigned Opc);
+struct VLdStIdxOpcodeInfo {
+ unsigned Opcode;
+ uint16_t BitWidth;
+ bool IsStore;
+};
+
+LLVM_READONLY
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByOpcode(unsigned Opc);
+
+LLVM_READONLY
+const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByKey(uint16_t BitWidth,
+ bool IsStore);
+
LLVM_READONLY
int getMTBUFBaseOpcode(unsigned Opc);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
new file mode 100644
index 0000000000000..08d3095de3561
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
@@ -0,0 +1,384 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of the VGPR "as memory" address space (13) on a
+; movrel-capable subtarget (gfx12). A load/store of a uniform (SGPR) pointer
+; lowers to an M0-relative move (v_movrels_b32 / v_movreld_b32) over the wave's
+; vector registers, with the dword index (pointer >> 2) placed in M0.
+
+define i32 @load_i32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define i64 @load_i64(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %p
+ %y = add i64 %x, 1
+ ret i64 %y
+}
+
+define <2 x float> @load_v2f32(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_v2f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_add_f32 v0, 0x42280000, v0 :: v_dual_add_f32 v1, 0x42280000, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x float>, ptr addrspace(13) %p
+ %y = fadd <2 x float> %x, <float 42.0, float 42.0>
+ ret <2 x float> %y
+}
+
+define <3 x float> @load_v3f32(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_v3f32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 64
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_v3f32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 64
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_add_f32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %p.2 = getelementptr i32, ptr addrspace(13) %p, i32 16
+ %x = load <3 x float>, ptr addrspace(13) %p
+ %y = load <3 x float>, ptr addrspace(13) %p.2
+ %z = fadd <3 x float> %x, %y
+ ret <3 x float> %z
+}
+
+define void @store_i32(ptr addrspace(13) inreg %p, i32 %x) {
+; GFX12-LABEL: store_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i64(ptr addrspace(13) inreg %p, i64 %x, i64 %y) {
+; GFX12-LABEL: store_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %z = add i64 %x, %y
+ store i64 %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v3i32(ptr addrspace(13) inreg %p, <3 x i32> %x, <3 x i32> %y) {
+; GFX12-SDAG-LABEL: store_v3i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v3i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v3
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = add <3 x i32> %x, %y
+ store <3 x i32> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_v8f16(ptr addrspace(13) inreg %p, <8 x half> %x, <8 x half> %y) {
+; GFX12-SDAG-LABEL: store_v8f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-SDAG-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-SDAG-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-SDAG-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_v8f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_pk_add_f16 v0, v0, v4
+; GFX12-GISEL-NEXT: v_pk_add_f16 v1, v1, v5
+; GFX12-GISEL-NEXT: v_pk_add_f16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_pk_add_f16 v3, v3, v7
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %z = fadd <8 x half> %x, %y
+ store <8 x half> %z, ptr addrspace(13) %p
+ ret void
+}
+
+define void @copy_i32(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %src
+ store i32 %x, ptr addrspace(13) %dst
+ ret void
+}
+
+define void @copy_v2i32_unaligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_v2i32_unaligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x i32>, ptr addrspace(13) %src, align 4
+ store <2 x i32> %x, ptr addrspace(13) %dst, align 4
+ ret void
+}
+
+define void @copy_i64_aligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) inreg %src) {
+; GFX12-LABEL: copy_i64_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s1, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: v_movreld_b32_e32 v1, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i64, ptr addrspace(13) %src, align 8
+ store i64 %x, ptr addrspace(13) %dst, align 8
+ ret void
+}
+
+; Null and poison pointers must be accepted (produce valid code) rather than
+; crash or fail the machine verifier. The specific null-pointer value is
+; defined by the parent change that introduces the address space.
+
+define i32 @load_null() {
+; GFX12-LABEL: load_null:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) null
+ ret i32 %x
+}
+
+define void @store_null(i32 %v) {
+; GFX12-LABEL: store_null:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 0
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) null
+ ret void
+}
+
+define i32 @load_poison() {
+; GFX12-SDAG-LABEL: load_poison:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_poison:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) poison
+ ret i32 %x
+}
+
+define void @store_poison(i32 %v) {
+; GFX12-SDAG-LABEL: store_poison:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_poison:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) poison
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
new file mode 100644
index 0000000000000..48d90ecaf2eff
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -0,0 +1,461 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Copy from VGPR "as memory" (address space 13) to global memory across the
+; range of legal whole-dword access sizes (32 up to 1024 bits). Each uniform
+; load lowers to a sequence of per-dword M0-relative moves (v_movrels_b32).
+
+define void @copy_i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %in
+ store i32 %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v2i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v2i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <2 x i32>, ptr addrspace(13) %in
+ store <2 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v3i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v3i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-NEXT: global_store_b96 v3, v[0:2], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <3 x i32>, ptr addrspace(13) %in
+ store <3 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v4i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-LABEL: copy_v4i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load <4 x i32>, ptr addrspace(13) %in
+ store <4 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v5i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v5i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b32 v5, v4, s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v5i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v5, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b32 v5, v4, s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <5 x i32>, ptr addrspace(13) %in
+ store <5 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v6i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v6i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v6i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b64 v6, v[4:5], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <6 x i32>, ptr addrspace(13) %in
+ store <6 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v7i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v7i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b96 v7, v[4:6], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v7, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v7i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v7, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v7, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b96 v7, v[4:6], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <7 x i32>, ptr addrspace(13) %in
+ store <7 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v8i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v8i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v8i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: s_clause 0x1
+; GFX12-GISEL-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <8 x i32>, ptr addrspace(13) %in
+ store <8 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v16i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v16i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-SDAG-NEXT: s_clause 0x3
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:48
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v16i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v16, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-GISEL-NEXT: s_clause 0x3
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:48
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <16 x i32>, ptr addrspace(13) %in
+ store <16 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v32i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v32i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v16, v16
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v17, v17
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v18, v18
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v19, v19
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v20, v20
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v21, v21
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v22, v22
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v23, v23
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v24, v24
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v25, v25
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v26, v26
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v27, v27
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v28, v28
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v29, v29
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v30, v30
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v31, v31
+; GFX12-SDAG-NEXT: s_clause 0x7
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v32, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v32i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v32, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v12, v12
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v13, v13
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v14, v14
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v15, v15
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v16, v16
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v17, v17
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v18, v18
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v19, v19
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v20, v20
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v21, v21
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v22, v22
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v23, v23
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v24, v24
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v25, v25
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v26, v26
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v27, v27
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v28, v28
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v29, v29
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v30, v30
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v31, v31
+; GFX12-GISEL-NEXT: s_clause 0x7
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX12-GISEL-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <32 x i32>, ptr addrspace(13) %in
+ store <32 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
new file mode 100644
index 0000000000000..28aa0ff7f0fb9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; A divergent (per-lane) dword index into VGPR "as memory" (address space 13)
+; is handled with a waterfall loop: for each unique index across the wave, set
+; M0 and do the M0-relative move under a matching-lane EXEC subset. The pointer
+; arrives in a VGPR (no inreg), so the index (pointer >> 2) is divergent.
+
+define i32 @load_i32(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define void @store_i32(ptr addrspace(13) %p, i32 %x) {
+; GFX12-SDAG-LABEL: store_i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
new file mode 100644
index 0000000000000..393928d359f96
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; A constant VGPR "as memory" (address space 13) pointer formed with inttoptr:
+; the dword index (256 >> 2 = 64) is a compile-time constant, so M0 is set from
+; that constant and the access is a single M0-relative move.
+
+define i32 @load_i32() {
+; GFX12-LABEL: load_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 m0, 64
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %p = inttoptr i32 256 to ptr addrspace(13)
+ %x = load i32, ptr addrspace(13) %p
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define void @store_i32(i32 %x) {
+; GFX12-LABEL: store_i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: s_mov_b32 m0, 64
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %p = inttoptr i32 256 to ptr addrspace(13)
+ %y = add i32 %x, 1
+ store i32 %y, ptr addrspace(13) %p
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
new file mode 100644
index 0000000000000..40263e1c975ad
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -0,0 +1,31 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
+; not yet implemented. They must be rejected with a clean diagnostic on both
+; SelectionDAG and GlobalISel, rather than failing with "cannot select" /
+; "unable to legalize".
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 89f72e559ba52..846c7b938f5fd 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,6 +68,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-assign-idx-to-m0
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -215,6 +216,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-assign-idx-to-m0
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -402,6 +404,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-assign-idx-to-m0
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index bb7ed3b58f8af..77092eb28d4bf 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Assign Idx To M0
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -354,6 +355,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -683,6 +685,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -1016,6 +1019,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Assign Idx To M0
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1365,6 +1369,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Assign Idx To M0
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From 40c7050a404c02de09e247cc536e5c8cafc36556 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 15 Jul 2026 14:25:19 -0500
Subject: [PATCH 02/19] Waterfall any non-SGPR VGPR-memory index, not just
virtual
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c648da68f0f94..3d67b583cadd7 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7719,8 +7719,10 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
// loop that executes the access once per unique index across the wave.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
MachineOperand *Idx = &LdStIdx->getIdxOp();
- if (Idx->isReg() && Idx->getReg().isVirtual() &&
- !RI.isSGPRClass(MRI.getRegClass(Idx->getReg())))
+ // Waterfall any non-SGPR index. isSGPRReg handles both virtual and physical
+ // registers, so a physical (non-SGPR) index - not expected here, but still
+ // possible - is made uniform rather than silently skipped.
+ if (Idx->isReg() && !RI.isSGPRReg(MRI, Idx->getReg()))
CreatedBB = generateWaterFallLoop(*this, MI, {Idx}, MDT);
return CreatedBB;
}
>From f42c82c3d28716f6a2b12893959ddf87f8bbc4ff Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 15 Jul 2026 14:36:32 -0500
Subject: [PATCH 03/19] Do not set a kill flag on the M0 index copy in
AMDGPUAssignIdxToM0
---
llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index 8c5de37eb46e5..3bc6709b7c197 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -55,10 +55,11 @@ static bool assignIdxToM0(MachineFunction &MF) {
MI.removeOperand(DefIdx);
// Add a copy from the index register to M0 and rewrite MI to read M0.
+ // No kill flag is set on the M0 use: kill flags are deprecated and are a
+ // no-op on the reserved M0 register.
BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
.add(IdxOp);
IdxOp.setReg(AMDGPU::M0);
- IdxOp.setIsKill();
Changed = true;
}
}
>From 4f6bcc96ba9ee2f8f1a344415b951fd0499d72b1 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 16 Jul 2026 09:07:52 -0500
Subject: [PATCH 04/19] Mask the VGPR-memory movrel base into the addressable
range
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index b7e26e2376354..54580c5a34d5a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -401,13 +401,14 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
unsigned Offset = LdSt.getOffsetOp().getImm();
unsigned NumDwords = LdSt.getBitWidth() / 32;
- // A statically out-of-range dword offset would fold into a base VGPR outside
- // the addressable register file - an out-of-bounds access of the VGPR
- // "as memory" (address space 13) region. When the whole file is addressable
- // the index is allowed to wrap; otherwise it must stay in range.
-#ifndef NDEBUG
+ // A statically out-of-range dword offset is an out-of-bounds (undefined
+ // behavior) access of the VGPR "as memory" (address space 13) region. Rather
+ // than diagnose it or emit an invalid register, mask the base into the
+ // addressable VGPR range below so the access is accepted and verifier-clean,
+ // matching the downstream implementation.
unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
+#ifndef NDEBUG
bool AllowOffsetWrap =
NumAddressableVGPRs == AMDGPU::IsaInfo::getTotalNumVGPRs(*ST);
assert((AllowOffsetWrap || Offset + NumDwords <= NumAddressableVGPRs) &&
@@ -418,9 +419,11 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
// The dword index is (M0 + $offset). Fold $offset into the base register so
- // each dword i reads/writes VGPR($offset + i) relative to M0.
+ // each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
+ // into the addressable range so a statically out-of-bounds offset still
+ // resolves to a valid register.
for (unsigned i = 0; i < NumDwords; ++i) {
- Register Base = AMDGPU::VGPR0 + Offset + i;
+ Register Base = AMDGPU::VGPR0 + ((Offset + i) & (NumAddressableVGPRs - 1));
Register Sub = Data;
if (NumDwords != 1)
Sub = TRI->getSubReg(Data, TRI->getSubRegFromChannel(i));
>From 27c1bd4c76e4e3934e0d89a437ecad837a78469a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 28 Jul 2026 00:35:27 +0300
Subject: [PATCH 05/19] Simplify demanded bits
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 23 +++++++++
.../as-vgpr-index-demanded-bits.ll | 51 +++++++++++++++++++
3 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 54580c5a34d5a..151c238759059 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -404,8 +404,7 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
// A statically out-of-range dword offset is an out-of-bounds (undefined
// behavior) access of the VGPR "as memory" (address space 13) region. Rather
// than diagnose it or emit an invalid register, mask the base into the
- // addressable VGPR range below so the access is accepted and verifier-clean,
- // matching the downstream implementation.
+ // addressable VGPR range below so the access is accepted and verifier-clean.
unsigned NumAddressableVGPRs = ST->getAddressableNumVGPRs(
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
#ifndef NDEBUG
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 48b6689769913..27442eed2defd 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19237,6 +19237,29 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
return performInsertVectorEltCombine(N, DCI);
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
+ case AMDGPUISD::REG_LOAD:
+ case AMDGPUISD::REG_STORE: {
+ const SIMachineFunctionInfo *MFI =
+ DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
+ unsigned NumAddressableVGPRs =
+ Subtarget->getAddressableNumVGPRs(MFI->getDynamicVGPRBlockSize());
+ APInt IndexMask =
+ APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
+
+ unsigned IndexOpIdx = 0;
+ switch (N->getOpcode()) {
+ case AMDGPUISD::REG_LOAD:
+ IndexOpIdx = 1;
+ break;
+ case AMDGPUISD::REG_STORE:
+ IndexOpIdx = 2;
+ break;
+ }
+
+ if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
+ return SDValue(N, 0);
+ break;
+ }
case ISD::LOAD: {
if (SDValue Widened = widenLoad(cast<LoadSDNode>(N), DCI))
return Widened;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
new file mode 100644
index 0000000000000..1a2689f23a1c4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; The VGPR "as memory" (address space 13) dword index only needs enough bits to
+; address all addressable VGPRs, so a redundant high-bit mask feeding the index
+; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
+; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
+; path the mask must not survive into the M0 index computation.
+
+define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
+; GFX12-SDAG-LABEL: load_masked_index:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_masked_index:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %idx = and i32 %arg, 65535
+ %ptr = inttoptr i32 %idx to ptr addrspace(13)
+ %v = load i32, ptr addrspace(13) %ptr
+ ret i32 %v
+}
+
+define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
+; GFX12-SDAG-LABEL: store_masked_index:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: store_masked_index:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_endpgm
+ %idx = and i32 %arg, 65535
+ %ptr = inttoptr i32 %idx to ptr addrspace(13)
+ store i32 %val, ptr addrspace(13) %ptr
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From 315d7ba6a6c633f9ba258eb8e0446a8572fe6819 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:56:31 +0300
Subject: [PATCH 06/19] Run AMDGPUAssignIdxToM0 for optnone functions
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 5 ++-
.../AddressSpaceVGPR/as-vgpr-optnone.ll | 43 +++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index 3bc6709b7c197..f2ddf2b5d9bb1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -76,8 +76,9 @@ class AMDGPUAssignIdxToM0Legacy : public MachineFunctionPass {
AMDGPUAssignIdxToM0Legacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: without the copy to M0
+ // the movrel that AMDGPULowerVGPREncoding emits later reads a stale index.
+ // It therefore must not be skipped for optnone functions.
return assignIdxToM0(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
new file mode 100644
index 0000000000000..cd48756741766
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+
+; AMDGPUAssignIdxToM0 is required lowering rather than an optimization: the
+; v_movrel that AMDGPULowerVGPREncoding emits reads the dword index from M0, so
+; without the copy to M0 it reads a stale value. The pass must therefore run
+; even for optnone functions - clang marks every function optnone at -O0 - so
+; the index below has to end up in M0 and not in a plain SGPR.
+
+define i32 @load_i32_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-LABEL: load_i32_optnone:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s1, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i32, ptr addrspace(13) %p
+ ret i32 %x
+}
+
+define void @store_i32_optnone(ptr addrspace(13) inreg %p, i32 %v) noinline optnone {
+; GFX12-LABEL: store_i32_optnone:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s1, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) %p
+ ret void
+}
>From 8816683a3991a8a33fa296f7aeaa4e3999823fe2 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 00:08:17 +0300
Subject: [PATCH 07/19] Stop miscompiling VGPR-memory accesses on subtargets
without movrel
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 4 +
.../AMDGPU/AMDGPUInstructionSelector.cpp | 13 +--
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 50 +++++++--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 21 ++--
llvm/lib/Target/AMDGPU/SIInstructions.td | 7 +-
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 101 ++++++++++++++++++
6 files changed, 165 insertions(+), 31 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index f2ddf2b5d9bb1..b8004ce9925ed 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -29,6 +29,10 @@ using namespace llvm;
static bool assignIdxToM0(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+
+ // Only movrel takes its index from M0. Subtargets without it index with the
+ // VGPR indexing mode instead, which AMDGPULowerVGPREncoding enables around
+ // the move with s_set_gpr_idx_on, reading the index straight out of its SGPR.
if (!ST.hasMovrel())
return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 4646d48e3e8db..3a4cde5c1c708 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -4503,14 +4503,11 @@ bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
if (!selectImpl(I, *CoverageInfo))
return false;
- // On a movrel subtarget the selected V_LOAD_IDX / V_STORE_IDX expands to an
- // M0-relative move (see AMDGPUAssignIdxToM0 and AMDGPULowerVGPREncoding). Add
- // an implicit-def of $m0: it records that the eventual move clobbers M0, and
- // - because an instruction defining a physical register is not hoisted/sunk -
- // keeps a divergent access pinned inside its waterfall loop.
- // AMDGPUAssignIdxToM0 removes it when it writes M0 for real.
- if (!Subtarget->hasMovrel())
- return true;
+ // The selected V_LOAD_IDX / V_STORE_IDX expands to an M0-relative move (see
+ // AMDGPULowerVGPREncoding), which clobbers M0 whether it indexes with movrel
+ // or with the VGPR indexing mode. Add an implicit-def of $m0 to record that,
+ // and - because an instruction defining a physical register is not
+ // hoisted/sunk - to keep a divergent access pinned inside its waterfall loop.
auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
if (LdStIdx->getIdxOp().isReg())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 151c238759059..68e9070e3678a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -49,6 +49,7 @@
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/bit.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
@@ -181,10 +182,11 @@ class AMDGPULowerVGPREncoding {
bool runOnMachineInstr(MachineInstr &MI);
/// Lower a VGPR "as memory" (address space 13) indexed load/store pseudo
- /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of M0-relative moves
- /// (v_movrels_b32 for loads, v_movreld_b32 for stores) over the wave's vector
- /// registers. M0 must already hold the dword index (see AMDGPUAssignIdxToM0).
- /// This replaces the pseudo, which is erased.
+ /// (V_LOAD_IDX_B<N> / V_STORE_IDX_B<N>) into a sequence of indexed moves over
+ /// the wave's vector registers: v_movrels_b32 / v_movreld_b32 where the
+ /// subtarget has movrel, and v_mov_b32 wrapped in s_set_gpr_idx_on/off where
+ /// it indexes with the VGPR indexing mode. This replaces the pseudo, which is
+ /// erased.
void lowerLoadStoreIdx(MachineInstr &MI);
/// Compute the mode for a single \p MI given \p Ops operands
@@ -395,9 +397,9 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
const DebugLoc &DL = MI.getDebugLoc();
const bool IsStore = LdSt.mayStore();
- // $data is operand 0 of both the load (def) and store (use) pseudos; M0
- // already holds the dword index (see AMDGPUAssignIdxToM0).
+ // $data is operand 0 of both the load (def) and store (use) pseudos.
Register Data = LdSt.getDataOp().getReg();
+ MachineOperand &IdxOp = LdSt.getIdxOp();
unsigned Offset = LdSt.getOffsetOp().getImm();
unsigned NumDwords = LdSt.getBitWidth() / 32;
@@ -414,8 +416,32 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
"out of bounds VGPR 'as memory' (address space 13) access");
#endif
- unsigned Opcode =
- IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+ // Subtargets with movrel take the index from M0, which AMDGPUAssignIdxToM0
+ // has already copied it into. The rest have no movrel and index with the VGPR
+ // indexing mode instead: s_set_gpr_idx_on enables it for one operand of the
+ // moves that follow, reading the index straight out of the SGPR holding it,
+ // so no copy is needed there.
+ const bool UseGPRIdxMode = ST->useVGPRIndexMode();
+
+ MachineInstr *SetOn = nullptr;
+ if (UseGPRIdxMode) {
+ SetOn = BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
+ .add(IdxOp)
+ .addImm(IsStore ? AMDGPU::VGPRIndexMode::DST_ENABLE
+ : AMDGPU::VGPRIndexMode::SRC0_ENABLE)
+ .getInstr();
+ SetOn->getOperand(3).setIsUndef();
+ } else {
+ assert(IdxOp.isReg() && IdxOp.getReg() == AMDGPU::M0 &&
+ "movrel index should have been copied into M0");
+ }
+
+ unsigned Opcode;
+ if (UseGPRIdxMode)
+ Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
+ : AMDGPU::V_MOV_B32_indirect_read;
+ else
+ Opcode = IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
// The dword index is (M0 + $offset). Fold $offset into the base register so
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
@@ -445,6 +471,14 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
runOnMachineInstr(*Mov);
}
+ // Keep the mode switch and the moves it applies to together, so nothing is
+ // scheduled or spilled in between while indexing is enabled.
+ if (SetOn) {
+ MachineInstr *SetOff =
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
+ finalizeBundle(BB, SetOn->getIterator(), std::next(SetOff->getIterator()));
+ }
+
MI.eraseFromParent();
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 27442eed2defd..1411eb3d10108 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19737,19 +19737,16 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
- // A VGPR "as memory" indexed load/store with a register index expands (on
- // movrel subtargets) to an M0-relative move. Add an implicit-def of $m0: it
- // records that the eventual move clobbers M0, and - because an instruction
- // defining a physical register is not hoisted/sunk - keeps a divergent access
- // pinned inside its waterfall loop. AMDGPUAssignIdxToM0 removes this when it
- // writes M0 for real.
+ // A VGPR "as memory" indexed load/store with a register index expands to an
+ // M0-relative move, which clobbers M0 whether it indexes with movrel or with
+ // the VGPR indexing mode. Add an implicit-def of $m0: it records that, and -
+ // because an instruction defining a physical register is not hoisted/sunk -
+ // keeps a divergent access pinned inside its waterfall loop.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
- if (getSubtarget()->hasMovrel()) {
- MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg())
- MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- }
+ MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg())
+ MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
return;
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 968b9f24f4803..929d9f9fd0219 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1070,9 +1070,10 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
// a 32-bit value that may be uniform (SGPR) or divergent (VGPR); $offset is a
// constant dword offset folded in at selection.
//
-// AMDGPUAssignIdxToM0 copies $idx into M0 before register allocation, then
-// AMDGPULowerVGPREncoding lowers each into v_movrels_b32 (load) /
-// v_movreld_b32 (store) over the wave's vector registers.
+// AMDGPULowerVGPREncoding lowers each into an M0-relative move over the wave's
+// vector registers: v_movrels_b32 (load) / v_movreld_b32 (store) where the
+// subtarget has movrel, and a v_mov_b32 under the VGPR indexing mode otherwise.
+// It writes $idx to M0 there, beside the move that reads it.
// Populates the VLdStIdxOpcodeInfo searchable table (see AMDGPUMachineInstrs.h
// and AMDGPU::getVLdStIdxOpcodeInfo*), mapping each pseudo to its bit width and
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
new file mode 100644
index 0000000000000..cebc6ea23c83b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+
+; The VGPR "as memory" address space (13) on a subtarget that has no movrel.
+; gfx9 indexes with the VGPR indexing mode instead, so AMDGPULowerVGPREncoding
+; wraps the move in s_set_gpr_idx_on / s_set_gpr_idx_off, which takes the dword
+; index straight from the SGPR holding it. Nothing needs to copy the index into
+; M0, so AMDGPUAssignIdxToM0 does nothing on these subtargets.
+;
+; The mode switch and the moves it applies to are bundled, so nothing can be
+; scheduled or spilled between them while indexing is enabled.
+
+define i32 @load_i32(ptr addrspace(13) inreg %p) {
+; GFX9-LABEL: load_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %v = load i32, ptr addrspace(13) %p, align 4
+ ret i32 %v
+}
+
+define void @store_i32(ptr addrspace(13) inreg %p, i32 %v) {
+; GFX9-LABEL: store_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ store i32 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; More than one dword: every move has to sit inside the same mode switch.
+define i64 @load_i64(ptr addrspace(13) inreg %p) {
+; GFX9-LABEL: load_i64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_lshr_b32 s0, s0, 2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, v1
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %v = load i64, ptr addrspace(13) %p, align 8
+ ret i64 %v
+}
+
+; A divergent pointer still needs the waterfall loop to make the index uniform
+; before the mode switch can use it.
+define i32 @load_i32_divergent(ptr addrspace(13) %p) {
+; GFX9-SDAG-LABEL: load_i32_divergent:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-SDAG-NEXT: ; %bb.2:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: load_i32_divergent:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: s_nop 1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
+; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_set_gpr_idx_off
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-GISEL-NEXT: ; %bb.2:
+; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = load i32, ptr addrspace(13) %p, align 4
+ ret i32 %v
+}
>From 6231187a7ceabb5d4aa94ae95c336c234232d4f7 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 02:15:42 +0300
Subject: [PATCH 08/19] Declare in TableGen that the VGPR-memory pseudos write
M0
---
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 11 ++------
.../AMDGPU/AMDGPUInstructionSelector.cpp | 24 ----------------
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 13 ---------
llvm/lib/Target/AMDGPU/SIInstructions.td | 13 +++++----
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 15 +++++-----
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 28 +++++++++----------
7 files changed, 32 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index b8004ce9925ed..f3972100574bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -51,14 +51,9 @@ static bool assignIdxToM0(MachineFunction &MF) {
assert(!MI.isBundled());
- // Remove the implicit-def $m0 that instruction selection added (to pin a
- // divergent access inside its waterfall loop); M0 is written for real
- // below.
- int DefIdx = MI.findRegisterDefOperandIdx(AMDGPU::M0, /*TRI=*/nullptr);
- assert(DefIdx >= 0);
- MI.removeOperand(DefIdx);
-
- // Add a copy from the index register to M0 and rewrite MI to read M0.
+ // Add a copy from the index register to M0 and rewrite MI to read M0. The
+ // pseudo goes on declaring that it writes M0: it stands for the whole
+ // sequence, and this copy is the write it describes.
// No kill flag is set on the M0 use: kill flags are deprecated and are a
// no-op on the reserved M0 register.
BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 3a4cde5c1c708..893c7cce1a0f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -15,7 +15,6 @@
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
-#include "AMDGPUMachineInstrs.h"
#include "AMDGPURegisterBankInfo.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -4496,26 +4495,6 @@ bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
return true;
}
-bool AMDGPUInstructionSelector::selectRegLoadStore(MachineInstr &I) const {
- // Remember where the selected machine instruction will land.
- MachineBasicBlock::iterator II = std::next(I.getIterator());
-
- if (!selectImpl(I, *CoverageInfo))
- return false;
-
- // The selected V_LOAD_IDX / V_STORE_IDX expands to an M0-relative move (see
- // AMDGPULowerVGPREncoding), which clobbers M0 whether it indexes with movrel
- // or with the VGPR indexing mode. Add an implicit-def of $m0 to record that,
- // and - because an instruction defining a physical register is not
- // hoisted/sunk - to keep a divergent access pinned inside its waterfall loop.
-
- auto *LdStIdx = cast<AMDGPUMI::VLoadStoreIdxInst>(&*std::prev(II));
- if (LdStIdx->getIdxOp().isReg())
- LdStIdx->addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- return true;
-}
-
bool AMDGPUInstructionSelector::select(MachineInstr &I) {
if (!I.isPreISelOpcode()) {
@@ -4651,9 +4630,6 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
return selectBVHIntersectRayIntrinsic(I);
- case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE:
- return selectRegLoadStore(I);
case AMDGPU::G_SBFX:
case AMDGPU::G_UBFX:
return selectG_SBFX_UBFX(I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 98f0cf2c3d009..ce3a090651d0e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -91,7 +91,6 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
bool selectCOPY_VCC_SCC(MachineInstr &I) const;
bool selectReadAnyLane(MachineInstr &I) const;
bool selectPHI(MachineInstr &I) const;
- bool selectRegLoadStore(MachineInstr &I) const;
bool selectG_TRUNC(MachineInstr &I) const;
bool selectG_SZA_EXT(MachineInstr &I) const;
bool selectG_FPEXT(MachineInstr &I) const;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1411eb3d10108..7ab15b349e440 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19737,19 +19737,6 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
- // A VGPR "as memory" indexed load/store with a register index expands to an
- // M0-relative move, which clobbers M0 whether it indexes with movrel or with
- // the VGPR indexing mode. Add an implicit-def of $m0: it records that, and -
- // because an instruction defining a physical register is not hoisted/sunk -
- // keeps a divergent access pinned inside its waterfall loop.
- if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
- MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg())
- MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
- /*isImp=*/true));
- return;
- }
-
if (TII->isImage(MI))
TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 929d9f9fd0219..6ca2e93b024da 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1089,10 +1089,11 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
VReg_512, VReg_1024] in {
// The units of $idx and $offset are in dwords.
//
- // VALU adds an implicit $exec use; together with the implicit-def $m0 added
- // by AdjustInstrPostInstrSelection (see hasPostISelHook), this keeps a
- // divergent-index access pinned inside its waterfall loop rather than being
- // hoisted/sunk out.
+ // The index reaches the hardware through M0, so these define it: either by
+ // writing M0 for v_movrel[sd], or through s_set_gpr_idx_on where the subtarget
+ // indexes with the VGPR indexing mode. Declaring that here also keeps a
+ // divergent-index access pinned inside its waterfall loop, since an
+ // instruction defining a physical register is not hoisted or sunk.
def V_LOAD_IDX_B#rc.Size : VPseudoInstSI <
(outs rc:$data),
(ins SReg_32:$idx, i32imm:$offset)>,
@@ -1101,7 +1102,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let hasPostISelHook = 1;
+ let Defs = [M0];
}
def V_STORE_IDX_B#rc.Size : VPseudoInstSI <
(outs),
@@ -1111,7 +1112,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let hasPostISelHook = 1;
+ let Defs = [M0];
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 28aa0ff7f0fb9..0cde03f7ab74c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -47,19 +47,20 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s1, v0
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index cebc6ea23c83b..1d3de91fd8bc2 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -60,40 +60,40 @@ define i32 @load_i32_divergent(ptr addrspace(13) %p) {
; GFX9-SDAG-LABEL: load_i32_divergent:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; GFX9-SDAG-NEXT: s_nop 1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-SDAG-NEXT: s_set_gpr_idx_off
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: load_i32_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-GISEL-NEXT: s_nop 1
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-GISEL-NEXT: s_set_gpr_idx_off
-; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-GISEL-NEXT: ; %bb.2:
+; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-GISEL-NEXT: s_set_gpr_idx_off
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
>From f5d5e003dcfbcd241f4f19ed910d16d32529fb29 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 15:26:49 +0300
Subject: [PATCH 09/19] Explain the undef operands of the VGPR-memory indexed
moves
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 14 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 224 ++++++++++++++++++
2 files changed, 237 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 68e9070e3678a..0cf4436583d8a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -447,6 +447,18 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
// into the addressable range so a statically out-of-bounds offset still
// resolves to a valid register.
+ //
+ // A move touches VGPR($offset + i) *plus M0*, which is only known at run
+ // time, so no operand can name the register it really reads or writes.
+ // Operands that name registers as memory rather than a value are therefore
+ // marked undef: the base of every move below, and the stored value as well
+ // when that is itself undef. Liveness of the registers behind this address
+ // space is consequently not expressed here, and correctness relies on nothing
+ // else being allocated to them - which is why frontend use of the address
+ // space is documented as discouraged.
+ const RegState DataFlags = IsStore
+ ? getUndefRegState(LdSt.getDataOp().isUndef())
+ : RegState::NoFlags;
for (unsigned i = 0; i < NumDwords; ++i) {
Register Base = AMDGPU::VGPR0 + ((Offset + i) & (NumAddressableVGPRs - 1));
Register Sub = Data;
@@ -457,7 +469,7 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
if (IsStore)
Mov = BuildMI(BB, MI, DL, TII->get(Opcode))
.addReg(Base, RegState::Undef)
- .addReg(Sub)
+ .addReg(Sub, DataFlags)
.getInstr();
else
Mov = BuildMI(BB, MI, DL, TII->get(Opcode), Sub)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
index 48d90ecaf2eff..34a8d3c2ceb9e 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -282,6 +282,230 @@ define void @copy_v8i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in
ret void
}
+define void @copy_v9i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v9i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b32 v9, v8, s[0:1] offset:32
+; GFX12-SDAG-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v9i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b32 v9, v8, s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <9 x i32>, ptr addrspace(13) %in
+ store <9 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v10i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v10i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v10, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v10, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v10, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b64 v10, v[8:9], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v10i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v10, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v10, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v10, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b64 v10, v[8:9], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <10 x i32>, ptr addrspace(13) %in
+ store <10 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v11i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v11i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v11, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v11, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b96 v11, v[8:10], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v11i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v11, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v11, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v11, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b96 v11, v[8:10], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <11 x i32>, ptr addrspace(13) %in
+ store <11 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
+define void @copy_v12i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
+; GFX12-SDAG-LABEL: copy_v12i32:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v12, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[4:7], s[0:1] offset:16
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[0:3], s[0:1]
+; GFX12-SDAG-NEXT: global_store_b128 v12, v[8:11], s[0:1] offset:32
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: copy_v12i32:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v12, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v3, v3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v4, v4
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v5, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v6, v6
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v7, v7
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v8, v8
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v9, v9
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v10, v10
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v11, v11
+; GFX12-GISEL-NEXT: s_clause 0x2
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[0:3], s[0:1]
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[4:7], s[0:1] offset:16
+; GFX12-GISEL-NEXT: global_store_b128 v12, v[8:11], s[0:1] offset:32
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load <12 x i32>, ptr addrspace(13) %in
+ store <12 x i32> %x, ptr addrspace(1) %out
+ ret void
+}
+
define void @copy_v16i32(ptr addrspace(1) inreg %out, ptr addrspace(13) inreg %in) {
; GFX12-SDAG-LABEL: copy_v16i32:
; GFX12-SDAG: ; %bb.0:
>From 4e9b3c777b9abccca3dc906e1c2b6132b195cd28 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Thu, 30 Jul 2026 17:25:11 +0300
Subject: [PATCH 10/19] Give the VGPR-memory moves their own opcodes
---
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 3 ++-
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 7 ++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 24 ++++++++-----------
llvm/lib/Target/AMDGPU/SIInstructions.td | 15 ++++++++++++
4 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 0cf4436583d8a..64a9e121bb63f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -441,7 +441,8 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
: AMDGPU::V_MOV_B32_indirect_read;
else
- Opcode = IsStore ? AMDGPU::V_MOVRELD_B32_e32 : AMDGPU::V_MOVRELS_B32_e32;
+ Opcode =
+ IsStore ? AMDGPU::V_MOVRELD_B32_as_mem : AMDGPU::V_MOVRELS_B32_as_mem;
// The dword index is (M0 + $offset). Fold $offset into the base register so
// each dword i reads/writes VGPR($offset + i) relative to M0. Mask the offset
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 41b93f49499ac..fbcdbf88c0b65 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -247,6 +247,13 @@ void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
Opcode == AMDGPU::V_FMA_MIX_BF16_t16) {
lowerT16FmaMixFP16(MI, OutMI);
return;
+ } else if (Opcode == AMDGPU::V_MOVRELS_B32_as_mem) {
+ // Indexed accesses of the VGPR "as memory" address space use their own
+ // opcodes because they index the register file rather than a tuple; they
+ // encode as the movrel they are named after.
+ Opcode = AMDGPU::V_MOVRELS_B32_e32;
+ } else if (Opcode == AMDGPU::V_MOVRELD_B32_as_mem) {
+ Opcode = AMDGPU::V_MOVRELD_B32_e32;
}
int MCOpcode = TII->pseudoToMCOpcode(Opcode);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3d67b583cadd7..462c011f52c47 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5844,8 +5844,6 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
- const bool IsPreRA = !MI.getMF()->getProperties().hasNoVRegs();
-
const unsigned StaticNumOps =
Desc.getNumOperands() + Desc.implicit_uses().size();
const unsigned NumImplicitOps = IsDst ? 2 : 1;
@@ -5854,7 +5852,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// post RA scheduler where the main implicit operand is killed and
// implicit-defs are added for sub-registers that remain live after this
// instruction.
- if (IsPreRA && MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
+ if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
ErrInfo = "missing implicit register operands";
return false;
}
@@ -5867,22 +5865,20 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
unsigned UseOpIdx;
- if (IsPreRA && (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
- UseOpIdx != StaticNumOps + 1)) {
+ if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
+ UseOpIdx != StaticNumOps + 1) {
ErrInfo = "movrel implicit operands should be tied";
return false;
}
}
- if (IsPreRA) {
- const MachineOperand &Src0 = MI.getOperand(Src0Idx);
- const MachineOperand &ImpUse =
- MI.getOperand(StaticNumOps + NumImplicitOps - 1);
- if (!ImpUse.isReg() || !ImpUse.isUse() ||
- !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
- ErrInfo = "src0 should be subreg of implicit vector use";
- return false;
- }
+ const MachineOperand &Src0 = MI.getOperand(Src0Idx);
+ const MachineOperand &ImpUse =
+ MI.getOperand(StaticNumOps + NumImplicitOps - 1);
+ if (!ImpUse.isReg() || !ImpUse.isUse() ||
+ !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
+ ErrInfo = "src0 should be subreg of implicit vector use";
+ return false;
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 6ca2e93b024da..5cf9cc65337d4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1116,6 +1116,21 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
}
}
+// Copies of v_movrel[sd]_b32 for the moves the pseudos above expand into.
+//
+// The storage a movrel indexes is normally a register tuple, and the machine
+// verifier requires an implicit use of that tuple naming the register the
+// instruction really touches. Here the storage is the whole register file,
+// which no operand can name, so these carry their own opcodes and leave those
+// rules to the tuple form. AMDGPUMCInstLower maps them back for encoding.
+let VALU = 1, VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
+ Size = V_MOV_B32_e32.Size in {
+ def V_MOVRELS_B32_as_mem
+ : VPseudoInstSI<(outs VGPR_32:$vdst), (ins VRegSrc_32:$src0)>;
+ def V_MOVRELD_B32_as_mem
+ : VPseudoInstSI<(outs), (ins VGPR_32:$vdst, VSrc_b32:$src0)>;
+}
+
// Select the REG_LOAD/REG_STORE target nodes into the sized indexed pseudos.
// The pointer is a byte offset into the file; SIreg_load/SIreg_store carry the
// dword index (ptr >> 2), and an (add idx, imm) shape folds a constant dword
>From 08d622f6d27f2af3702fe19ccf39e3b5c47f186c Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:58:56 +0300
Subject: [PATCH 11/19] Pin VGPR-memory indexed accesses to EXEC and mark them
divergent
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 10 +++----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++++++++
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 15 +++++-----
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 28 +++++++++----------
4 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a99a4c67afcb3..8d1175400c160 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1683,7 +1683,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
// so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
// than failing to legalize.
- Actions.customIf([=](const LegalityQuery &Query) -> bool {
+ Actions.customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3511,9 +3511,9 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
- const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
- auto Two = B.buildConstant(I32, 2);
- const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+ const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ MachineInstrBuilder Two = B.buildConstant(I32, 2);
+ const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
// Normalize the value to i32 / <N x i32> so a selection pattern always
// exists (e.g. for v4i8).
@@ -3533,7 +3533,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
.addMemOperand(&MMO);
} else {
- const auto Result =
+ const MachineInstrBuilder Result =
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
.addMemOperand(&MMO);
B.buildBitcast(ValReg, Result);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 462c011f52c47..9c052b1b195ea 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -264,6 +264,13 @@ bool SIInstrInfo::isReMaterializableImpl(
bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
assert(isVALU(MI, /*AllowLDSDMA=*/true));
+ // A VGPR "as memory" indexed access reads or writes the per-lane vector
+ // registers of the active lanes, so which lanes are active is part of what it
+ // does. Its implicit use of EXEC must not be treated as ignorable, or the
+ // access could be moved across a write to EXEC.
+ if (isa<AMDGPUMI::VLoadStoreIdxInst>(MI))
+ return true;
+
// If it is convergent it depends on EXEC.
if (MI.isConvergent())
return true;
@@ -11321,6 +11328,12 @@ ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
return ValueUniformity::Default;
}
+ // As above for the generic opcodes, but after instruction selection: an
+ // indexed load reads the wave's per-lane view of its vector registers, so
+ // even a uniform index yields a divergent value.
+ if (isa<AMDGPUMI::VLoadIdxInst>(MI))
+ return ValueUniformity::NeverUniform;
+
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 0cde03f7ab74c..28aa0ff7f0fb9 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -47,20 +47,19 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s1, v0
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-GISEL-NEXT: ; %bb.2:
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index 1d3de91fd8bc2..cebc6ea23c83b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -60,40 +60,40 @@ define i32 @load_i32_divergent(ptr addrspace(13) %p) {
; GFX9-SDAG-LABEL: load_i32_divergent:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-SDAG-NEXT: s_nop 1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-SDAG-NEXT: s_set_gpr_idx_off
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: load_i32_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-GISEL-NEXT: s_nop 1
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9-GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
-; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
-; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
; GFX9-GISEL-NEXT: s_set_gpr_idx_off
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
>From 3c860eba3c6ac3ba05a9a3912e1f1bb8accf3cbf Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 3 Aug 2026 19:49:32 +0300
Subject: [PATCH 12/19] Address review: subarch triples, required pass mixin,
legalizer predicates, redundant VALU
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 ++-
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 5 +++--
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 21 +++++++++----------
llvm/lib/Target/AMDGPU/SIInstructions.td | 4 +---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll | 4 ++--
.../AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 8 +++----
.../as-vgpr-index-demanded-bits.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-inttoptr.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-optnone.ll | 4 ++--
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 4 ++--
12 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index d50ea0b22a6a1..bd68ed70ae02e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -446,7 +446,8 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
-class AMDGPUAssignIdxToM0Pass : public PassInfoMixin<AMDGPUAssignIdxToM0Pass> {
+class AMDGPUAssignIdxToM0Pass
+ : public RequiredPassInfoMixin<AMDGPUAssignIdxToM0Pass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
index f3972100574bf..2c651a2d70727 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
@@ -45,9 +45,10 @@ static bool assignIdxToM0(MachineFunction &MF) {
if (!LdSt)
continue;
+ // The operand class of the index is a register class, so it never holds
+ // an immediate that would have to be moved into M0 separately.
MachineOperand &IdxOp = LdSt->getIdxOp();
- if (!IdxOp.isReg())
- continue;
+ assert(IdxOp.isReg() && "VGPR-memory index must be a register");
assert(!MI.isBundled());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 8d1175400c160..c15b54efbcd1a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -458,6 +458,11 @@ static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
return false;
+ // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
+ // to G_AMDGPU_REG_LOAD/STORE.
+ if (AS == AMDGPUAS::VGPR)
+ return false;
+
// Do not handle extending vector loads.
if (Ty.isVector() && MemSize != RegSize)
return false;
@@ -550,10 +555,6 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
- // VGPR ("as memory") accesses are never plain-legal; they are custom-lowered
- // to G_AMDGPU_REG_LOAD/STORE.
- if (Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR)
- return false;
return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
@@ -722,6 +723,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
+ const LLT VGPRPtr = GetAddrSpacePtr(AMDGPUAS::VGPR);
const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER);
const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE);
const LLT BufferStridedPtr =
@@ -1675,17 +1677,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
// 64-bits.
//
- // TODO: Should generalize bitcast action into coerce, which will also cover
- // inserting addrspacecasts.
- Actions.customIf(typeIs(1, Constant32Ptr));
-
// VGPR ("as memory") accesses are custom-lowered to the legal
// G_AMDGPU_REG_LOAD/STORE target instructions. Always take the custom path
// so an unsupported (e.g. sub-dword) access is diagnosed cleanly rather
// than failing to legalize.
- Actions.customIf([](const LegalityQuery &Query) -> bool {
- return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
- });
+ //
+ // TODO: Should generalize bitcast action into coerce, which will also cover
+ // inserting addrspacecasts.
+ Actions.customIf(typeInSet(1, {Constant32Ptr, VGPRPtr}));
// Turn any illegal element vectors into something easier to deal
// with. These will ultimately produce 32-bit scalar shifts to extract the
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 5cf9cc65337d4..fb9ef1f492e97 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1099,7 +1099,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
(ins SReg_32:$idx, i32imm:$offset)>,
VLdStIdxOpcodeInfo<rc.Size, 0> {
let mayLoad = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1109,7 +1108,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
(ins rc:$data, SReg_32:$idx, i32imm:$offset)>,
VLdStIdxOpcodeInfo<rc.Size, 1> {
let mayStore = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1123,7 +1121,7 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
// instruction really touches. Here the storage is the whole register file,
// which no operand can name, so these carry their own opcodes and leave those
// rules to the tuple form. AMDGPUMCInstLower maps them back for encoding.
-let VALU = 1, VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
+let VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
Size = V_MOV_B32_e32.Size in {
def V_MOVRELS_B32_as_mem
: VPseudoInstSI<(outs VGPR_32:$vdst), (ins VRegSrc_32:$src0)>;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
index 08d3095de3561..5ff034e4d1893 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; End-to-end lowering of the VGPR "as memory" address space (13) on a
; movrel-capable subtarget (gfx12). A load/store of a uniform (SGPR) pointer
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
index 34a8d3c2ceb9e..d4c916671796c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-copy.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Copy from VGPR "as memory" (address space 13) to global memory across the
; range of legal whole-dword access sizes (32 up to 1024 bits). Each uniform
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 28aa0ff7f0fb9..4dd453b87bbf2 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; A divergent (per-lane) dword index into VGPR "as memory" (address space 13)
; is handled with a waterfall loop: for each unique index across the wave, set
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index cebc6ea23c83b..ffb673f547dd4 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX9,GFX9-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.0a-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.0a-- -filetype=null %s
; The VGPR "as memory" address space (13) on a subtarget that has no movrel.
; gfx9 indexes with the VGPR indexing mode instead, so AMDGPULowerVGPREncoding
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 1a2689f23a1c4..e1dfb50df7fb9 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
index 393928d359f96..ae759fa010e7a 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-inttoptr.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; A constant VGPR "as memory" (address space 13) pointer formed with inttoptr:
; the dword index (256 >> 2 = 64) is a compile-time constant, so M0 is set from
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
index cd48756741766..5d25e4522447b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-optnone.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12
; AMDGPUAssignIdxToM0 is required lowering rather than an optimization: the
; v_movrel that AMDGPULowerVGPREncoding emits reads the dword index from M0, so
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 40263e1c975ad..75a744f49f10f 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,5 +1,5 @@
-; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
-; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
; not yet implemented. They must be rejected with a clean diagnostic on both
>From d4a2f792f5b0ac997863bcd99eb413f77091315d Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 01:38:17 +0300
Subject: [PATCH 13/19] Set up M0 for VGPR-memory accesses in finalizeLowering
instead of a separate pass
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 10 --
.../lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp | 110 ------------------
.../Target/AMDGPU/AMDGPULowerVGPREncoding.cpp | 11 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 -
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 --
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 34 ++++++
llvm/lib/Target/AMDGPU/SIInstructions.td | 9 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 -
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 -
10 files changed, 47 insertions(+), 146 deletions(-)
delete mode 100644 llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index bd68ed70ae02e..2036ec03e25d6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -229,9 +229,6 @@ extern char &AMDGPURegBankLegalizeID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
-void initializeAMDGPUAssignIdxToM0LegacyPass(PassRegistry &);
-extern char &AMDGPUAssignIdxToM0ID;
-
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -446,13 +443,6 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
-class AMDGPUAssignIdxToM0Pass
- : public RequiredPassInfoMixin<AMDGPUAssignIdxToM0Pass> {
-public:
- PreservedAnalyses run(MachineFunction &MF,
- MachineFunctionAnalysisManager &MFAM);
-};
-
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
deleted file mode 100644
index 2c651a2d70727..0000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUAssignIdxToM0.cpp
+++ /dev/null
@@ -1,110 +0,0 @@
-//===- AMDGPUAssignIdxToM0.cpp - Copy VGPR-memory indices to M0 ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// Copy the register index of a VGPR "as memory" (address space 13)
-/// V_LOAD_IDX / V_STORE_IDX pseudo into M0, which V_MOVREL[SD] reads when the
-/// pseudo is lowered (see AMDGPULowerVGPREncoding). This runs before register
-/// allocation so the copy to M0 is inserted while the index is still virtual.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPU.h"
-#include "AMDGPUMachineInstrs.h"
-#include "GCNSubtarget.h"
-#include "SIInstrInfo.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachinePassManager.h"
-#include "llvm/InitializePasses.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "amdgpu-assign-idx-to-m0"
-
-static bool assignIdxToM0(MachineFunction &MF) {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-
- // Only movrel takes its index from M0. Subtargets without it index with the
- // VGPR indexing mode instead, which AMDGPULowerVGPREncoding enables around
- // the move with s_set_gpr_idx_on, reading the index straight out of its SGPR.
- if (!ST.hasMovrel())
- return false;
-
- const SIInstrInfo *TII = ST.getInstrInfo();
-
- bool Changed = false;
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
- if (!LdSt)
- continue;
-
- // The operand class of the index is a register class, so it never holds
- // an immediate that would have to be moved into M0 separately.
- MachineOperand &IdxOp = LdSt->getIdxOp();
- assert(IdxOp.isReg() && "VGPR-memory index must be a register");
-
- assert(!MI.isBundled());
-
- // Add a copy from the index register to M0 and rewrite MI to read M0. The
- // pseudo goes on declaring that it writes M0: it stands for the whole
- // sequence, and this copy is the write it describes.
- // No kill flag is set on the M0 use: kill flags are deprecated and are a
- // no-op on the reserved M0 register.
- BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
- .add(IdxOp);
- IdxOp.setReg(AMDGPU::M0);
- Changed = true;
- }
- }
-
- return Changed;
-}
-
-namespace {
-
-class AMDGPUAssignIdxToM0Legacy : public MachineFunctionPass {
-public:
- static char ID;
-
- AMDGPUAssignIdxToM0Legacy() : MachineFunctionPass(ID) {}
-
- bool runOnMachineFunction(MachineFunction &MF) override {
- // This is required lowering, not an optimization: without the copy to M0
- // the movrel that AMDGPULowerVGPREncoding emits later reads a stale index.
- // It therefore must not be skipped for optnone functions.
- return assignIdxToM0(MF);
- }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
- MachineFunctionPass::getAnalysisUsage(AU);
- }
-
- StringRef getPassName() const override { return "AMDGPU Assign Idx To M0"; }
-};
-
-} // end anonymous namespace
-
-PreservedAnalyses
-AMDGPUAssignIdxToM0Pass::run(MachineFunction &MF,
- MachineFunctionAnalysisManager &MFAM) {
- if (!assignIdxToM0(MF))
- return PreservedAnalyses::all();
- auto PA = getMachineFunctionPassPreservedAnalyses();
- PA.preserveSet<CFGAnalyses>();
- return PA;
-}
-
-char AMDGPUAssignIdxToM0Legacy::ID = 0;
-
-char &llvm::AMDGPUAssignIdxToM0ID = AMDGPUAssignIdxToM0Legacy::ID;
-
-INITIALIZE_PASS(AMDGPUAssignIdxToM0Legacy, DEBUG_TYPE,
- "AMDGPU Assign Idx To M0", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 64a9e121bb63f..9046c17b57d8a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -416,11 +416,12 @@ void AMDGPULowerVGPREncoding::lowerLoadStoreIdx(MachineInstr &MI) {
"out of bounds VGPR 'as memory' (address space 13) access");
#endif
- // Subtargets with movrel take the index from M0, which AMDGPUAssignIdxToM0
- // has already copied it into. The rest have no movrel and index with the VGPR
- // indexing mode instead: s_set_gpr_idx_on enables it for one operand of the
- // moves that follow, reading the index straight out of the SGPR holding it,
- // so no copy is needed there.
+ // Subtargets with movrel take the index from M0, which
+ // SITargetLowering::finalizeLowering has already copied it into. The rest
+ // have no movrel and index with the VGPR indexing mode instead:
+ // s_set_gpr_idx_on enables it for one operand of the moves that follow,
+ // reading the index straight out of the SGPR holding it, so no copy is needed
+ // there.
const bool UseGPRIdxMode = ST->useVGPRIndexMode();
MachineInstr *SetOn = nullptr;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index af2b377df5eb4..6da139ea0b59c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -116,7 +116,6 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
#endif
MACHINE_FUNCTION_PASS("amdgpu-asm-printer", AMDGPUAsmPrinterPass())
-MACHINE_FUNCTION_PASS("amdgpu-assign-idx-to-m0", AMDGPUAssignIdxToM0Pass())
MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0bd83203446f5..544245875f8b7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -693,7 +693,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizePass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
- initializeAMDGPUAssignIdxToM0LegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1836,10 +1835,6 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
- // Copy the VGPR "as memory" load/store index into M0 before register
- // allocation; the movrel emitted later by AMDGPULowerVGPREncoding reads it.
- addPass(&AMDGPUAssignIdxToM0ID);
-
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
@@ -2650,10 +2645,6 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
- // Set up M0 for the movrel that expands a VGPR "as memory" indexed access.
- // Run before allocation so the index computation coalesces into M0.
- addMachineFunctionPass(AMDGPUAssignIdxToM0Pass(), PMW);
-
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
}
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 054fc1d14d7cb..b18ca6bd24040 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -47,7 +47,6 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUArgumentUsageInfo.cpp
AMDGPUAsanInstrumentation.cpp
AMDGPUAsmPrinter.cpp
- AMDGPUAssignIdxToM0.cpp
AMDGPUAtomicOptimizer.cpp
AMDGPUAttributor.cpp
AMDGPUBarrierLatency.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 7ab15b349e440..300d0c63e36c7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20201,6 +20201,40 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
Info->limitOccupancy(MF);
+ // A VGPR "as memory" indexed access takes its index from M0 on subtargets
+ // with movrel, so copy it there and rewrite the access to read M0, which lets
+ // the index computation coalesce into it. Doing this here rather than while
+ // selecting the access is what makes a divergent index correct: both
+ // selectors make such an index uniform with a waterfall loop - the register
+ // bank legalizer before selection, SIFixSGPRCopies after it - and this runs
+ // after both, so the copy lands inside the loop, next to the per-iteration
+ // index it has to carry. Subtargets without movrel index with the VGPR
+ // indexing mode instead, reading the index straight out of its SGPR.
+ //
+ // TODO: This is only needed because M0 is reserved. Once it is not, the index
+ // can be an ordinary virtual register copy that the coalescer folds away, and
+ // this can go.
+ if (ST.hasMovrel()) {
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
+ if (!LdSt)
+ continue;
+
+ MachineOperand &IdxOp = LdSt->getIdxOp();
+ assert(IdxOp.isReg() && "VGPR-memory index must be a register");
+ // With GlobalISel this runs twice, once from InstructionSelect and
+ // again from FinalizeISel; the rewrite makes the second run a no-op.
+ if (IdxOp.getReg() == AMDGPU::M0)
+ continue;
+
+ BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
+ .addReg(IdxOp.getReg());
+ IdxOp.setReg(AMDGPU::M0);
+ }
+ }
+ }
+
if (ST.isWave32() && !MF.empty()) {
for (auto &MBB : MF) {
for (auto &MI : MBB) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index fb9ef1f492e97..50a97fa2d48d1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1090,10 +1090,15 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
// The units of $idx and $offset are in dwords.
//
// The index reaches the hardware through M0, so these define it: either by
- // writing M0 for v_movrel[sd], or through s_set_gpr_idx_on where the subtarget
- // indexes with the VGPR indexing mode. Declaring that here also keeps a
+ // writing M0 for v_movrel[sd], which SITargetLowering::finalizeLowering
+ // rewrites $idx to, or through s_set_gpr_idx_on where the subtarget indexes
+ // with the VGPR indexing mode. Declaring that here also keeps a
// divergent-index access pinned inside its waterfall loop, since an
// instruction defining a physical register is not hoisted or sunk.
+ //
+ // Unlike SI_INDIRECT_SRC/DST and the V_INDIRECT_REG_* pseudos, the storage
+ // indexed here is the whole register file rather than a tuple an operand can
+ // name, so these are memory accesses carrying an MMO instead.
def V_LOAD_IDX_B#rc.Size : VPseudoInstSI <
(outs rc:$data),
(ins SReg_32:$idx, i32imm:$offset)>,
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 846c7b938f5fd..89f72e559ba52 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,7 +68,6 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
-; GCN-O0-NEXT: amdgpu-assign-idx-to-m0
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -216,7 +215,6 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
-; GCN-O2-NEXT: amdgpu-assign-idx-to-m0
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -404,7 +402,6 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
-; GCN-O3-NEXT: amdgpu-assign-idx-to-m0
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 77092eb28d4bf..bb7ed3b58f8af 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,7 +113,6 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
-; GCN-O0-NEXT: AMDGPU Assign Idx To M0
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -355,7 +354,6 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
-; GCN-O1-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -685,7 +683,6 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
-; GCN-O1-OPTS-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -1019,7 +1016,6 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
-; GCN-O2-NEXT: AMDGPU Assign Idx To M0
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1369,7 +1365,6 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
-; GCN-O3-NEXT: AMDGPU Assign Idx To M0
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From 067c428a7d6b9ee9be355201765e139e65d49ae9 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 14:36:35 +0300
Subject: [PATCH 14/19] Round-trip VGPR-memory pointers through flat via a
synthetic aperture
---
llvm/docs/AMDGPUUsage.rst | 5 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 34 +++-
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 4 +
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 9 +
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 4 +
llvm/lib/Target/AMDGPU/SIDefines.h | 12 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 29 +++-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +
.../AddressSpaceVGPR/as-vgpr-addrspacecast.ll | 164 ++++++++++++++++++
9 files changed, 252 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-addrspacecast.ll
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 5adc8b4299091..08a594fc70182 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1172,6 +1172,11 @@ supported for the ``amdgcn`` target.
aligned to 2^32 which makes it easier to convert from flat to segment or
segment to flat.
+ *Synthetic apertures* are defined that enable safe roundtrips of pointers
+ from special address spaces through the generic address space. Attempting to
+ dereference generic pointers obtained in this way (using e.g. ``load`` or
+ ``store``) has undefined behavior.
+
A global address space address has the same value when used as a flat address
so no conversion is needed.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c15b54efbcd1a..a8f9b954454c7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2444,10 +2444,30 @@ bool AMDGPULegalizerInfo::legalizeCustom(
llvm_unreachable("expected switch to return");
}
-Register AMDGPULegalizerInfo::getSegmentAperture(
- unsigned AS,
- MachineRegisterInfo &MRI,
- MachineIRBuilder &B) const {
+Register AMDGPULegalizerInfo::getSegmentAperture(unsigned AS,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) const {
+ // See SITargetLowering::getSegmentAperture: an address space without an
+ // aperture of its own round-trips through the generic address space using the
+ // shared aperture tagged with its synthetic aperture number.
+ unsigned BaseAS = AS;
+ unsigned SANum = AMDGPU::tryGetSyntheticApertureNumber(AS);
+ if (SANum != AMDGPU::SyntheticAperture::None)
+ BaseAS = AMDGPUAS::LOCAL_ADDRESS;
+
+ Register Aperture = getBaseSegmentAperture(BaseAS, MRI, B);
+
+ if (SANum != AMDGPU::SyntheticAperture::None) {
+ const LLT I32 = LLT::integer(32);
+ auto Tag = B.buildConstant(I32, SANum);
+ return B.buildOr(I32, Aperture, Tag).getReg(0);
+ }
+
+ return Aperture;
+}
+
+Register AMDGPULegalizerInfo::getBaseSegmentAperture(
+ unsigned AS, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const LLT I32 = LLT::integer(32);
@@ -2586,7 +2606,7 @@ bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
(DestAS == AMDGPUAS::LOCAL_ADDRESS ||
- DestAS == AMDGPUAS::PRIVATE_ADDRESS)) {
+ DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::VGPR)) {
auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
ST.hasGloballyAddressableScratch()) {
@@ -2631,8 +2651,8 @@ bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
}
if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
- (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
- SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
+ (SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::PRIVATE_ADDRESS ||
+ SrcAS == AMDGPUAS::VGPR)) {
auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
// Coerce the type of the low half of the result so we can use
// merge_values.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 89819fe990f5a..30fd16930e6ae 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -264,6 +264,10 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool legalizeIntrinsic(LegalizerHelper &Helper,
MachineInstr &MI) const override;
+
+private:
+ Register getBaseSegmentAperture(unsigned AS, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) const;
};
} // End llvm namespace.
#endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index a560df6f2d911..ed9d140ce40e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -74,6 +74,15 @@ static TargetExtType *getTargetExtType(const GlobalVariable &GV) {
}
}
+unsigned tryGetSyntheticApertureNumber(unsigned AS) {
+ switch (AS) {
+ case AMDGPUAS::VGPR:
+ return SyntheticAperture::VGPR;
+ default:
+ return SyntheticAperture::None;
+ }
+}
+
TargetExtType *isNamedBarrier(const GlobalVariable &GV) {
if (TargetExtType *Ty = getTargetExtType(GV))
return Ty->getName() == "amdgcn.named.barrier" ? Ty : nullptr;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 4e164d08549dd..f81b677051561 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -35,6 +35,10 @@ using VariableFunctionMap = DenseMap<GlobalVariable *, DenseSet<Function *>>;
Align getAlign(const DataLayout &DL, const GlobalVariable *GV);
+// Get the synthetic aperture number for the given address space, or None (0)
+// if the address space does not have one.
+unsigned tryGetSyntheticApertureNumber(unsigned AS);
+
// Copy metadata onto a load widened to read a superset of Source's bytes. Only
// value-independent metadata is copied; metadata describing the loaded value
// (!range, !noundef, !nofpclass, !tbaa, ...) is dropped.
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 9f2446072bf0b..562d68eabdb52 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -1367,6 +1367,18 @@ enum {
};
} // namespace Barrier
+
+namespace SyntheticAperture {
+enum SyntheticAperture {
+ None = 0,
+
+ NAMED_BARRIER = 1,
+ SEMAPHORES = 2,
+ VGPR = 3,
+ // The addition of new apertures must be coordinated with the architecture
+ // team
+};
+} // namespace SyntheticAperture
} // namespace AMDGPU
// clang-format off
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 300d0c63e36c7..13538ad8c4981 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -9317,6 +9317,27 @@ SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
SelectionDAG &DAG) const {
+ // An address space that has no aperture of its own round-trips through the
+ // generic address space using a synthetic aperture: the shared aperture with
+ // the aperture number in its low bits. Dereferencing such a generic pointer
+ // is undefined behaviour; the round-trip only has to preserve the value.
+ unsigned BaseAS = AS;
+ unsigned SANum = AMDGPU::tryGetSyntheticApertureNumber(AS);
+ if (SANum != AMDGPU::SyntheticAperture::None)
+ BaseAS = AMDGPUAS::LOCAL_ADDRESS;
+
+ SDValue Aperture = getBaseSegmentAperture(BaseAS, DL, DAG);
+
+ if (SANum != AMDGPU::SyntheticAperture::None) {
+ SDValue Tag = DAG.getConstant(SANum, DL, MVT::i32);
+ return DAG.getNode(ISD::OR, DL, MVT::i32, Aperture, Tag);
+ }
+
+ return Aperture;
+}
+
+SDValue SITargetLowering::getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
+ SelectionDAG &DAG) const {
if (Subtarget->hasApertureRegs()) {
const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
? AMDGPU::SRC_SHARED_BASE
@@ -9417,10 +9438,10 @@ SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
- // flat -> local/private
+ // flat -> local/private/vgpr
if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
- DestAS == AMDGPUAS::PRIVATE_ADDRESS) {
+ DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::VGPR) {
SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
@@ -9447,10 +9468,10 @@ SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
}
}
- // local/private -> flat
+ // local/private/vgpr -> flat
if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
- SrcAS == AMDGPUAS::PRIVATE_ADDRESS) {
+ SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::VGPR) {
SDValue CvtPtr;
if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
Subtarget->hasGloballyAddressableScratch()) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 4aab0a9ba4d01..685f15717ce2d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -183,6 +183,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue lowerXMULO(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
+ SDValue getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
+ SelectionDAG &DAG) const;
SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-addrspacecast.ll
new file mode 100644
index 0000000000000..05efaddedc1e1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-addrspacecast.ll
@@ -0,0 +1,164 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-GISEL
+
+; A pointer to the VGPR "as memory" address space (13) round-trips through the
+; generic address space using a synthetic aperture: the shared aperture with the
+; aperture number in its low bits. The round-trip preserves the value, including
+; the -1 null pointer, but the generic pointer must not be dereferenced.
+
+define ptr @vgpr_to_flat(ptr addrspace(13) %ptr) {
+; GFX12-SDAG-LABEL: vgpr_to_flat:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; GFX12-SDAG-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_or_b32 s0, s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: vgpr_to_flat:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; GFX12-GISEL-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_or_b32 s0, s1, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-SDAG-LABEL: vgpr_to_flat:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_or_b32 s0, s1, 3
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: vgpr_to_flat:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_or_b32 s0, s1, 3
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, s0, vcc_lo
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %flat = addrspacecast ptr addrspace(13) %ptr to ptr
+ ret ptr %flat
+}
+
+define ptr addrspace(13) @flat_to_vgpr(ptr %flat) {
+; GFX12-LABEL: flat_to_vgpr:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: flat_to_vgpr:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = addrspacecast ptr %flat to ptr addrspace(13)
+ ret ptr addrspace(13) %v
+}
+
+; With the pointer known to be non-null the null checks are gone.
+
+define ptr @vgpr_to_flat_nonnull(ptr addrspace(13) %ptr) {
+; GFX12-LABEL: vgpr_to_flat_nonnull:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, s1, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: vgpr_to_flat_nonnull:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], src_shared_base
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_or_b32 s0, s1, 3
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %flat = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p13(ptr addrspace(13) %ptr)
+ ret ptr %flat
+}
+
+define ptr addrspace(13) @flat_to_vgpr_nonnull(ptr %flat) {
+; GFX12-LABEL: flat_to_vgpr_nonnull:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: flat_to_vgpr_nonnull:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call ptr addrspace(13) @llvm.amdgcn.addrspacecast.nonnull.p13.p0(ptr %flat)
+ ret ptr addrspace(13) %v
+}
+
+define ptr addrspace(13) @vgpr_roundtrip(ptr addrspace(13) %ptr) {
+; GFX12-LABEL: vgpr_roundtrip:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: vgpr_roundtrip:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %flat = addrspacecast ptr addrspace(13) %ptr to ptr
+ %v = addrspacecast ptr %flat to ptr addrspace(13)
+ ret ptr addrspace(13) %v
+}
>From 5ecb4a45847272229278794a9621727d046d4ef3 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 17:13:17 +0300
Subject: [PATCH 15/19] Declare M0 on the VGPR-memory pseudos only where the
expansion writes it
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 69 +++++++++++--------
llvm/lib/Target/AMDGPU/SIInstructions.td | 16 ++---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll | 23 +++++++
3 files changed, 70 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 13538ad8c4981..1f8fd0857e6b4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20222,37 +20222,48 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
Info->limitOccupancy(MF);
- // A VGPR "as memory" indexed access takes its index from M0 on subtargets
- // with movrel, so copy it there and rewrite the access to read M0, which lets
- // the index computation coalesce into it. Doing this here rather than while
- // selecting the access is what makes a divergent index correct: both
- // selectors make such an index uniform with a waterfall loop - the register
- // bank legalizer before selection, SIFixSGPRCopies after it - and this runs
- // after both, so the copy lands inside the loop, next to the per-iteration
- // index it has to carry. Subtargets without movrel index with the VGPR
- // indexing mode instead, reading the index straight out of its SGPR.
+ // Give a VGPR "as memory" indexed access its M0 operand, matching how
+ // AMDGPULowerVGPREncoding will expand it. Under the VGPR indexing mode the
+ // expansion emits an s_set_gpr_idx_on, which reads the index out of its SGPR
+ // and clobbers M0, so the access only has to declare that clobber. With
+ // movrel the index has to be in M0, so copy it there and rewrite the access
+ // to read M0, which lets the index computation coalesce into the copy.
//
- // TODO: This is only needed because M0 is reserved. Once it is not, the index
- // can be an ordinary virtual register copy that the coalescer folds away, and
- // this can go.
- if (ST.hasMovrel()) {
- for (MachineBasicBlock &MBB : MF) {
- for (MachineInstr &MI : MBB) {
- auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
- if (!LdSt)
- continue;
-
- MachineOperand &IdxOp = LdSt->getIdxOp();
- assert(IdxOp.isReg() && "VGPR-memory index must be a register");
- // With GlobalISel this runs twice, once from InstructionSelect and
- // again from FinalizeISel; the rewrite makes the second run a no-op.
- if (IdxOp.getReg() == AMDGPU::M0)
- continue;
-
- BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
- .addReg(IdxOp.getReg());
- IdxOp.setReg(AMDGPU::M0);
+ // Doing this here rather than while selecting the access is what makes a
+ // divergent index correct: both selectors make such an index uniform with a
+ // waterfall loop - the register bank legalizer before selection,
+ // SIFixSGPRCopies after it - and this runs after both, so the copy lands
+ // inside the loop, next to the per-iteration index it has to carry.
+ //
+ // TODO: The copy is only needed because M0 is reserved. Once it is not, the
+ // index can be an ordinary virtual register copy that the coalescer folds
+ // away, and this can go.
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI);
+ if (!LdSt)
+ continue;
+
+ // With GlobalISel this runs twice, once from InstructionSelect and again
+ // from FinalizeISel, so both forms below have to be idempotent.
+ if (ST.useVGPRIndexMode()) {
+ if (!MI.definesRegister(AMDGPU::M0, TRI))
+ MI.addOperand(MachineOperand::CreateReg(AMDGPU::M0, /*isDef=*/true,
+ /*isImp=*/true));
+ continue;
}
+
+ MachineOperand &IdxOp = LdSt->getIdxOp();
+ assert(IdxOp.isReg() && "VGPR-memory index must be a register");
+ if (IdxOp.getReg() == AMDGPU::M0)
+ continue;
+
+ BuildMI(MBB, &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::M0)
+ .addReg(IdxOp.getReg());
+ IdxOp.setReg(AMDGPU::M0);
+ // M0 is reserved, and the value copied into it can still be there for a
+ // later access, so this read must not kill it.
+ IdxOp.setIsKill(false);
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 50a97fa2d48d1..18e2f3097c431 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1073,7 +1073,6 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
// AMDGPULowerVGPREncoding lowers each into an M0-relative move over the wave's
// vector registers: v_movrels_b32 (load) / v_movreld_b32 (store) where the
// subtarget has movrel, and a v_mov_b32 under the VGPR indexing mode otherwise.
-// It writes $idx to M0 there, beside the move that reads it.
// Populates the VLdStIdxOpcodeInfo searchable table (see AMDGPUMachineInstrs.h
// and AMDGPU::getVLdStIdxOpcodeInfo*), mapping each pseudo to its bit width and
@@ -1089,12 +1088,13 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
VReg_512, VReg_1024] in {
// The units of $idx and $offset are in dwords.
//
- // The index reaches the hardware through M0, so these define it: either by
- // writing M0 for v_movrel[sd], which SITargetLowering::finalizeLowering
- // rewrites $idx to, or through s_set_gpr_idx_on where the subtarget indexes
- // with the VGPR indexing mode. Declaring that here also keeps a
- // divergent-index access pinned inside its waterfall loop, since an
- // instruction defining a physical register is not hoisted or sunk.
+ // The index reaches the hardware through M0, and how it gets there depends on
+ // the subtarget, so SITargetLowering::finalizeLowering fills that in rather
+ // than TableGen: for v_movrel[sd] it copies the index to M0 and rewrites $idx
+ // to it, making the access a plain M0 read; under the VGPR indexing mode the
+ // s_set_gpr_idx_on it expands to clobbers M0 instead, which the access
+ // carries as an implicit def. A divergent-index access is kept inside its
+ // waterfall loop by its implicit use of EXEC (see resultDependsOnExec).
//
// Unlike SI_INDIRECT_SRC/DST and the V_INDIRECT_REG_* pseudos, the storage
// indexed here is the whole register file rather than a tuple an operand can
@@ -1106,7 +1106,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let mayLoad = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
def V_STORE_IDX_B#rc.Size : VPseudoInstSI <
(outs),
@@ -1115,7 +1114,6 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
let mayStore = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
index 5ff034e4d1893..295149a115326 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-basic.ll
@@ -297,6 +297,29 @@ define void @copy_i64_aligned(ptr addrspace(13) inreg %dst, ptr addrspace(13) in
ret void
}
+; An access reads M0 rather than clobbering it, so two accesses at the same
+; index need it written only once.
+
+define i32 @load_i32_twice(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i32_twice:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load volatile i32, ptr addrspace(13) %p
+ %y = load volatile i32, ptr addrspace(13) %p
+ %z = add i32 %x, %y
+ ret i32 %z
+}
+
; Null and poison pointers must be accepted (produce valid code) rather than
; crash or fail the machine verifier. The specific null-pointer value is
; defined by the parent change that introduces the address space.
>From ee0b3d3ec134b8003ab54a0c198c72d9caffe291 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 13:08:10 -0400
Subject: [PATCH 16/19] Describe the high register bits of the VGPR-memory
indexed moves
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 7 ++-
.../as-vgpr-high-registers.ll | 53 +++++++++++++++++++
2 files changed, 59 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-high-registers.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 18e2f3097c431..7fb9e53b15330 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1124,8 +1124,13 @@ foreach rc = [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_160, VReg_192,
// instruction really touches. Here the storage is the whole register file,
// which no operand can name, so these carry their own opcodes and leave those
// rules to the tuple form. AMDGPUMCInstLower maps them back for encoding.
+// AMDGPULowerVGPREncoding reaches $vdst and $src0 by name to decide which high
+// address bits a subtarget with more than 256 addressable VGPRs needs, so these
+// have to be in the named operand table. Without it that lookup finds nothing,
+// no S_SET_VGPR_MSB is emitted, and a base register at or above 256 is encoded
+// as its low eight bits - a silent access of the wrong register.
let VOP1 = 1, Uses = [M0, EXEC], hasSideEffects = 0,
- Size = V_MOV_B32_e32.Size in {
+ UseNamedOperandTable = 1, Size = V_MOV_B32_e32.Size in {
def V_MOVRELS_B32_as_mem
: VPseudoInstSI<(outs VGPR_32:$vdst), (ins VRegSrc_32:$src0)>;
def V_MOVRELD_B32_as_mem
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-high-registers.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-high-registers.ll
new file mode 100644
index 0000000000000..75e726536fe58
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-high-registers.ll
@@ -0,0 +1,53 @@
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=CHECK,GISEL
+
+; A subtarget with more than 256 addressable VGPRs encodes a register number's
+; high bits separately, with S_SET_VGPR_MSB. A whole-dword access folds its
+; constant dword offset into the base register of each indexed move, so an
+; access whose base reaches v256 or beyond needs those bits described -
+; otherwise only the low eight bits are encoded and the move silently touches a
+; register 256 lower than the one meant.
+;
+; The moves must therefore be in the named operand table, since that is how
+; AMDGPULowerVGPREncoding finds the operands whose high bits it has to describe.
+;
+; Only SelectionDAG folds the offset into the base; GlobalISel folds it into the
+; index instead and indexes from v0, so it cannot reach a high base this way and
+; needs no mode change. Both are checked, because the difference is the reason
+; this went unnoticed.
+
+; The dword index is %i + 254, so a four-dword access spans v254, v255, v256 and
+; v257 relative to M0.
+; CHECK-LABEL: fold_across_256:
+; SDAG: v_movrels_b32_e32 v{{[0-9]+}}, v254
+; SDAG-NEXT: v_movrels_b32_e32 v{{[0-9]+}}, v255
+; SDAG-NEXT: s_set_vgpr_msb 1
+; SDAG-NEXT: v_movrels_b32_e32 v{{[0-9]+}}, v0
+; SDAG-NEXT: v_movrels_b32_e32 v{{[0-9]+}}, v1
+; SDAG: s_set_vgpr_msb 0x100
+;
+; GISEL: s_lshl2_add_u32 s0, s0, 0x3f8
+; GISEL: v_movrels_b32_e32 v{{[0-9]+}}, v0
+; GISEL-NOT: s_set_vgpr_msb
+define void @fold_across_256(ptr addrspace(1) %out, i32 inreg %i) {
+ %s = shl nuw i32 %i, 2
+ %a = add nuw i32 %s, 1016
+ %p = inttoptr i32 %a to ptr addrspace(13)
+ %v = load <4 x i32>, ptr addrspace(13) %p, align 16
+ store <4 x i32> %v, ptr addrspace(1) %out, align 16
+ ret void
+}
+
+; Entirely below 256, so neither path needs a mode change.
+; CHECK-LABEL: fold_below_256:
+; SDAG: v_movrels_b32_e32 v{{[0-9]+}}, v4
+; GISEL: v_movrels_b32_e32 v{{[0-9]+}}, v0
+; CHECK-NOT: s_set_vgpr_msb
+define void @fold_below_256(ptr addrspace(1) %out, i32 inreg %i) {
+ %s = shl nuw i32 %i, 2
+ %a = add nuw i32 %s, 16
+ %p = inttoptr i32 %a to ptr addrspace(13)
+ %v = load i32, ptr addrspace(13) %p, align 4
+ store i32 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
>From b161af9cb953caac0027593e54a1e205cba4779c Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 13:08:23 -0400
Subject: [PATCH 17/19] Treat a VGPR-memory load as divergent on SelectionDAG
too
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 16 +++++--
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 45 +++++++++++++++++++
2 files changed, 58 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1f8fd0857e6b4..0b919f250ab2b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20640,9 +20640,19 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
case ISD::LOAD: {
const LoadSDNode *L = cast<LoadSDNode>(N);
unsigned AS = L->getAddressSpace();
- // A flat load may access private memory.
- return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
- }
+ // A flat load may access private memory. A load of the VGPR "as memory"
+ // address space reads this lane's own registers, so it is divergent however
+ // uniform the index is - and it is still an ISD::LOAD until the pre-ISel
+ // combine turns it into a REG_LOAD below.
+ return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS ||
+ AS == AMDGPUAS::VGPR;
+ }
+ // The lowered form of the above. Without this the DAG takes the node's
+ // divergence to be that of its operands, so a uniform index makes the loaded
+ // value look uniform, and a consumer that requires a uniform operand gets a
+ // v_readfirstlane - broadcasting one lane's value to the whole wave.
+ case AMDGPUISD::REG_LOAD:
+ return true;
case ISD::CALLSEQ_END:
return true;
case ISD::INTRINSIC_WO_CHAIN:
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 4dd453b87bbf2..33ab5a46c37ba 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -126,5 +126,50 @@ define void @store_i32(ptr addrspace(13) %p, i32 %x) {
store i32 %y, ptr addrspace(13) %p
ret void
}
+; The value read out of the address space is per-lane whatever the index is, so
+; a uniform index does not make it uniform. A consumer that requires a uniform
+; operand must therefore not be handed it directly: doing so inserts a
+; readfirstlane, which broadcasts one lane's value across the whole wave.
+declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32 immarg)
+
+define i32 @uniform_index_divergent_value(ptr addrspace(13) inreg %p, <4 x i32> inreg %rsrc) {
+; GFX12-SDAG-LABEL: uniform_index_divergent_value:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX12-SDAG-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: uniform_index_divergent_value:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = load i32, ptr addrspace(13) %p, align 4
+ %r = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %rsrc, i32 %v, i32 0)
+ ret i32 %r
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
>From 0564959fff24b1a79ecef3000de6497866415862 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 16:02:25 -0400
Subject: [PATCH 18/19] Reject under-aligned whole-dword VGPR-memory accesses
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 17 +++++---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 19 +++++---
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 43 +++++++++++++++----
3 files changed, 60 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a8f9b954454c7..7a6b51c7a6549 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -3513,16 +3513,23 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic instead of failing to legalize
- // (sub-dword support lands in a later change).
+ // Only dword-aligned whole-dword, non-extending/non-truncating accesses are
+ // implemented. Reject anything else with a diagnostic instead of failing to
+ // legalize (sub-dword support lands in a later change).
+ //
+ // The alignment is checked here rather than in the size predicate: the index
+ // built below is the pointer shifted right by two, which discards the low two
+ // bits rather than accounting for them, so an under-aligned access would
+ // silently reach the dword containing the address instead of the bytes asked
+ // for. That is a property of how the address is formed, not of the size.
if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
- ValSize)) {
+ ValSize) ||
+ MMO.getAlign() < Align(4)) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword loads and stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0b919f250ab2b..37e64eff81471 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13494,17 +13494,17 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic (replacing the value with poison)
- // instead of failing instruction selection. Both callers - operation
- // legalization and the pre-ISel combine - replace the node with this result,
- // so the diagnostic is emitted exactly once.
+ // Only dword-aligned whole-dword, non-extending/non-truncating accesses are
+ // implemented. Reject anything else with a diagnostic (replacing the value
+ // with poison) instead of failing instruction selection. Both callers -
+ // operation legalization and the pre-ISel combine - replace the node with
+ // this result, so the diagnostic is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword loads and stores are implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13514,6 +13514,13 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
if (BitWidth < 32)
return reportUnsupported();
+
+ // The dword index built below is the pointer shifted right by two, which
+ // discards the low two bits rather than accounting for them, so an
+ // under-aligned access would silently reach the dword containing the address
+ // instead of the bytes asked for.
+ if (MemOp->getAlign() < Align(4))
+ return reportUnsupported();
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 75a744f49f10f..0cbda8e8ff646 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,31 +1,58 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
-; not yet implemented. They must be rejected with a clean diagnostic on both
-; SelectionDAG and GlobalISel, rather than failing with "cannot select" /
-; "unable to legalize".
+; Accesses of the VGPR "as memory" address space (13) that are not implemented
+; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
+; rather than failing with "cannot select" / "unable to legalize" - or, worse,
+; silently generating wrong code.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+; Sub-dword (8/16-bit) accesses are not yet implemented; support lands in a
+; later change.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
define i8 @load_i8(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
ret i8 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
define i16 @load_i16(ptr addrspace(13) inreg %p) {
%x = load i16, ptr addrspace(13) %p
ret i16 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
store i8 %v, ptr addrspace(13) %p
ret void
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
store i16 %v, ptr addrspace(13) %p
ret void
}
+
+; An access addresses registers by the dword index pointer >> 2, which discards
+; the low two bits rather than accounting for them. An under-aligned one would
+; therefore reach the dword containing the address instead of the bytes asked
+; for - the same code as a correctly aligned access, reading the wrong data with
+; nothing to show for it.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
+ %x = load i32, ptr addrspace(13) %p, align 1
+ ret i32 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
+ store i32 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
+; Alignment is required of the pointer, not of the accessed type: a 64-bit
+; access needs only the dword alignment the index computation relies on.
+; CHECK-NOT: in function load_i64_align4
+define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
+ %x = load i64, ptr addrspace(13) %p, align 4
+ ret i64 %x
+}
>From 1689a3dba7a648482c0289d49c3f544e7b9faf13 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 26 Aug 2026 14:43:11 -0400
Subject: [PATCH 19/19] Cover the divergent index on wave64 and pin the two
EXEC/uniformity guards
---
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 125 ++++++++++++++++++
llvm/unittests/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp | 125 ++++++++++++++++++
3 files changed, 251 insertions(+)
create mode 100644 llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 33ab5a46c37ba..e779e6ff4d334 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -1,11 +1,19 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX942,GFX942-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -o - %s | FileCheck %s --check-prefixes=GFX942,GFX942-GISEL
; A divergent (per-lane) dword index into VGPR "as memory" (address space 13)
; is handled with a waterfall loop: for each unique index across the wave, set
; M0 and do the M0-relative move under a matching-lane EXEC subset. The pointer
; arrives in a VGPR (no inreg), so the index (pointer >> 2) is divergent.
+;
+; gfx942 covers the two axes gfx1200 cannot. It is wave64, so the loop runs over
+; a 64-lane mask rather than a 32-lane one, and it indexes with the VGPR indexing
+; mode instead of movrel, so the index reaches the hardware by a different route.
+; gfx1250 cannot stand in for the first: it is wave32 only, and asking it for
+; wave64 makes llc emit no functions at all, which reads as a passing test.
define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-SDAG-LABEL: load_i32:
@@ -61,6 +69,48 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: load_i32:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX942-SDAG-NEXT: s_nop 1
+; GFX942-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX942-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-SDAG-NEXT: s_set_gpr_idx_off
+; GFX942-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX942-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_1
+; GFX942-SDAG-NEXT: ; %bb.2:
+; GFX942-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v1
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: load_i32:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_off
+; GFX942-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX942-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_1
+; GFX942-GISEL-NEXT: ; %bb.2:
+; GFX942-GISEL-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 1, v1
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
ret i32 %y
@@ -122,6 +172,50 @@ define void @store_i32(ptr addrspace(13) %p, i32 %x) {
; GFX12-GISEL-NEXT: ; %bb.2:
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: store_i32:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX942-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-SDAG-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX942-SDAG-NEXT: s_nop 1
+; GFX942-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX942-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(DST)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-SDAG-NEXT: s_set_gpr_idx_off
+; GFX942-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX942-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX942-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_1
+; GFX942-SDAG-NEXT: ; %bb.2:
+; GFX942-SDAG-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: store_i32:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX942-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(DST)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-GISEL-NEXT: s_set_gpr_idx_off
+; GFX942-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX942-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX942-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_1
+; GFX942-GISEL-NEXT: ; %bb.2:
+; GFX942-GISEL-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%y = add i32 %x, 1
store i32 %y, ptr addrspace(13) %p
ret void
@@ -166,6 +260,36 @@ define i32 @uniform_index_divergent_value(ptr addrspace(13) inreg %p, <4 x i32>
; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: uniform_index_divergent_value:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX942-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX942-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX942-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX942-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX942-SDAG-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX942-SDAG-NEXT: s_set_gpr_idx_off
+; GFX942-SDAG-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: uniform_index_divergent_value:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX942-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX942-GISEL-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, v0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_off
+; GFX942-GISEL-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
%r = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %rsrc, i32 %v, i32 0)
ret i32 %r
@@ -173,3 +297,4 @@ define i32 @uniform_index_divergent_value(ptr addrspace(13) inreg %p, <4 x i32>
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
+; GFX942: {{.*}}
diff --git a/llvm/unittests/Target/AMDGPU/CMakeLists.txt b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
index a2d0a0797b1fe..a2c40c978d900 100644
--- a/llvm/unittests/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
@@ -34,4 +34,5 @@ add_llvm_target_unittest(AMDGPUTests
PALMetadata.cpp
RCIUpdateReservedRegsTest.cpp
UniformityAnalysisTest.cpp
+ VGPRAsMemory.cpp
)
diff --git a/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
new file mode 100644
index 0000000000000..01e628a67e9f2
--- /dev/null
+++ b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
@@ -0,0 +1,125 @@
+//===--------- llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Two properties of the VGPR "as memory" (address space 13) indexed accesses
+// are asserted here rather than in a lit test, because no pass can be made to
+// observe them: these pseudos carry implicit-def $m0, so they define a physical
+// register, and MachineLICM, MachineSink and MachineCSE all decline to touch
+// them for that reason alone. Their safety today is therefore incidental, and
+// the properties below are what it would rest on if that incidental protection
+// ever went away.
+//
+// Each is paired with an ordinary VALU that must answer the other way, so a
+// change that made the query answer uniformly fails here rather than passing
+// vacuously.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUUnitTests.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MIRParser/MIRParser.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "gtest/gtest.h"
+
+#include "AMDGPUGenSubtargetInfo.inc"
+
+using namespace llvm;
+
+class VGPRAsMemoryTest : public AMDGPUCodeGenTestBase {
+public:
+ void SetUp() override { setUpImpl("amdgpu12.00-amd-", "", ""); }
+};
+
+// An indexed access reads or writes the per-lane vector registers of the active
+// lanes, so which lanes are active is part of what it does. Its implicit use of
+// EXEC must not be reported ignorable: that is what would otherwise let it be
+// hoisted or sunk across a write to EXEC, changing the set of lanes touched.
+TEST_F(VGPRAsMemoryTest, ExecUseIsNotIgnorable) {
+ StringRef MIRString = R"MIR(
+name: exec_use
+body: |
+ bb.0:
+ liveins: $m0, $vgpr0
+
+ $vgpr1 = V_LOAD_IDX_B32 $m0, 0, implicit $exec :: (load (s32), addrspace 13)
+ V_STORE_IDX_B32 $vgpr0, $m0, 0, implicit $exec :: (store (s32), addrspace 13)
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ S_ENDPGM 0
+...
+)MIR";
+
+ ASSERT_TRUE(parseMIR(MIRString));
+ MachineFunction &MF = getMF("exec_use");
+ const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
+ MachineBasicBlock *MBB = MF.getBlockNumbered(0);
+
+ auto ExecUseOf = [](const MachineInstr &MI) -> const MachineOperand * {
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
+ return &MO;
+ return nullptr;
+ };
+
+ for (MachineInstr &MI : *MBB) {
+ const MachineOperand *Exec = ExecUseOf(MI);
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_B32:
+ case AMDGPU::V_STORE_IDX_B32:
+ ASSERT_NE(Exec, nullptr) << "indexed access lost its implicit EXEC";
+ EXPECT_FALSE(TII->isIgnorableUse(MI, MI.getOperandNo(Exec)))
+ << "an indexed access may not be moved across a write to EXEC";
+ break;
+ case AMDGPU::V_MOV_B32_e32:
+ // The contrast: a plain lane-wise move produces the same value in every
+ // lane it writes, so its EXEC use really is ignorable.
+ ASSERT_NE(Exec, nullptr);
+ EXPECT_TRUE(TII->isIgnorableUse(MI, MI.getOperandNo(Exec)));
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+// An indexed load reads the wave's per-lane view of its vector registers, so
+// the value is divergent however the index was computed. Reporting it as
+// possibly-uniform would invite a readfirstlane, broadcasting one lane's value
+// across the wave.
+TEST_F(VGPRAsMemoryTest, IndexedLoadIsNeverUniform) {
+ StringRef MIRString = R"MIR(
+name: uniformity
+body: |
+ bb.0:
+ liveins: $m0
+
+ $vgpr0 = V_LOAD_IDX_B32 $m0, 0, implicit $exec :: (load (s32), addrspace 13)
+ $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ S_ENDPGM 0
+...
+)MIR";
+
+ ASSERT_TRUE(parseMIR(MIRString));
+ MachineFunction &MF = getMF("uniformity");
+ const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
+ MachineBasicBlock *MBB = MF.getBlockNumbered(0);
+
+ for (MachineInstr &MI : *MBB) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_B32:
+ EXPECT_EQ(TII->getValueUniformity(MI), ValueUniformity::NeverUniform);
+ break;
+ case AMDGPU::V_MOV_B32_e32:
+ // The contrast: an ordinary move is only as divergent as its operands.
+ EXPECT_EQ(TII->getValueUniformity(MI), ValueUniformity::Default);
+ break;
+ default:
+ break;
+ }
+ }
+}
More information about the llvm-branch-commits
mailing list