[llvm-branch-commits] [llvm] [AMDGPU] Add support for sub-dword types for address space 13 (PR #213223)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 14 06:05:45 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/213223
>From afcff3f6e62524d0b4d8930a1582aa760d254d6b Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 12:39:47 +0300
Subject: [PATCH 01/19] Add support for sub-dword types for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 10 +
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 2 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 146 +++++-
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 201 ++++++++
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 9 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 6 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 18 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 11 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 138 +++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 18 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 82 ++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 449 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 41 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
19 files changed, 1108 insertions(+), 51 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 48ba95b2a1a41..84791f744cf25 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -223,6 +223,9 @@ extern char &AMDGPURegBankLegalizeID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
+void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
+extern char &AMDGPULowerIdxOpsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -437,6 +440,13 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
+class AMDGPULowerIdxOpsPass
+ : public RequiredPassInfoMixin<AMDGPULowerIdxOpsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 38a692a45fa29..7143bb682a414 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -284,6 +284,8 @@ def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD_BITS, SIreg_load_bits>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE_BITS, SIreg_store_bits>;
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2be19b3965112..c23e510dab4c3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -557,12 +557,20 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Only whole-dword
-// accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported
-// for now; sub-dword (8/16-bit) support lands later.
+// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
+// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
+// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
- return MemSize == ValSize &&
- AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+ if (MemSize == 8 || MemSize == 16) {
+ if (ValSize == MemSize)
+ return true;
+ if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
+ return true;
+ return false;
+ }
+ if (MemSize != ValSize)
+ return false;
+ return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
/// Return true if a load or store of the type should be lowered with a bitcast
@@ -1812,6 +1820,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
{S16, ConstantPtr, S8, GlobalAlign8}})
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
+ })
+ // VGPR ("as memory") extending loads are custom-lowered to
+ // G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
+ // unsupported access is diagnosed cleanly rather than failing to
+ // legalize.
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
if (ST.hasFlatAddressSpace()) {
@@ -3492,18 +3507,21 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
-/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into a legal
-/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE indexed by the pointer's dword offset
-/// (pointer >> 2). Parallels the SelectionDAG LowerLoadStoreVGPR.
+/// Lower a G_LOAD / G_STORE / G_SEXTLOAD / G_ZEXTLOAD on AMDGPUAS::VGPR into a
+/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
+/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
+/// Parallels the SelectionDAG LowerLoadStoreVGPR.
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
MachineMemOperand &MMO = **MI.memoperands_begin();
- const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ const unsigned Opcode = MI.getOpcode();
+ const bool IsStore = Opcode == AMDGPU::G_STORE;
Register ValReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
+ const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
const LLT ValTy = MRI.getType(ValReg);
const unsigned ValSize = ValTy.getSizeInBits();
// The GISel selection patterns for the indexed pseudos - and for the shift /
@@ -3512,16 +3530,14 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic instead of failing to legalize
- // (sub-dword support lands in a later change).
- if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
- ValSize)) {
+ // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
+ // with a diagnostic instead of failing to legalize.
+ if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize)) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "whole-dword and 8-/16-bit loads and stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
@@ -3529,6 +3545,102 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (MemSize < 32) {
+ assert(MemSize == 8 || MemSize == 16);
+ assert(MemSize <= ValSize && ValSize <= 32);
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // known constant.
+ Register BaseReg = PtrReg;
+ int64_t Offset = 0;
+ if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
+ if (auto MaybeOff =
+ getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
+ BaseReg = PtrAdd->getBaseReg();
+ Offset = MaybeOff->Value.getSExtValue();
+ }
+ }
+
+ bool HaveConstantBitOffset = false;
+ int64_t ConstantBitOffsetVal = 0;
+ if (Offset == 0 && MMO.getAlign() >= Align(4)) {
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = 0;
+ } else {
+ auto &VT = *Helper.getValueTracking();
+ KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = (Offset & 3) * 8;
+ }
+ }
+
+ // Setup common registers.
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ Register BitOffsetReg;
+ if (HaveConstantBitOffset) {
+ BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
+ // offset, so masking the high bits is unnecessary.
+ auto Three = B.buildConstant(I32, 3);
+ BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
+ }
+
+ if (IsStore) {
+ LLT IValTy = LLT::integer(ValSize);
+ Register Value = ValReg;
+ if (IValTy != ValTy)
+ Value = B.buildBitcast(IValTy, Value).getReg(0);
+ if (ValSize != 32)
+ Value = B.buildAnyExt(I32, Value).getReg(0);
+
+ Register ShiftAmt = BitOffsetReg;
+ if (!HaveConstantBitOffset) {
+ auto Mask = B.buildConstant(I32, 31);
+ ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
+ }
+ Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
+
+ const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
+ const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
+ .addMemOperand(&MMO);
+ } else {
+ const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
+ const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ Register Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
+ {Index, BitWidthReg, BitOffsetReg, IsSExtReg})
+ .addMemOperand(&MMO)
+ .getReg(0);
+
+ if (ValTy != I32) {
+ LLT ResultTy = I32;
+ if (ValSize != 32) {
+ ResultTy = LLT::integer(ValSize);
+ Result = B.buildTrunc(ResultTy, Result).getReg(0);
+ }
+ if (ResultTy != ValTy)
+ Result = B.buildBitcast(ValTy, Result).getReg(0);
+ }
+
+ B.buildCopy(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+ }
+
const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
MachineInstrBuilder Two = B.buildConstant(I32, 2);
const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
@@ -3571,7 +3683,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
- if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ // G_LOAD as well as the extending loads (G_SEXTLOAD / G_ZEXTLOAD) are
+ // custom-lowered; the latter only exist for sub-dword accesses.
+ if (AddrSpace == AMDGPUAS::VGPR)
return lowerLoadStoreVGPR(Helper, MI);
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
new file mode 100644
index 0000000000000..95cfe42df7e6d
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -0,0 +1,201 @@
+//===- AMDGPULowerIdxOps.cpp - Expand sub-dword VGPR-memory accesses -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Expand the sub-dword VGPR "as memory" (address space 13) pseudos into a
+/// whole-dword indexed access plus a bit-field extract or insert:
+///
+/// V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+///
+/// A sub-dword store is therefore a read-modify-write of the containing dword.
+/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
+/// here take part in the usual M0 setup, and before register allocation
+/// because it introduces new virtual registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-lower-idx-ops"
+
+namespace {
+
+class LowerIdxOps {
+public:
+ LowerIdxOps(MachineFunction &MF)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ MRI(&MF.getRegInfo()) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ void lowerLoadIdxBits(MachineInstr &MI);
+ void lowerStoreIdxBits(MachineInstr &MI);
+
+ const SIInstrInfo *TII;
+ MachineRegisterInfo *MRI;
+};
+
+void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
+
+ const bool IsSigned = MI.getOperand(5).getImm() != 0;
+ const MCInstrDesc &II =
+ TII->get(IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPUMI::VLoadIdxInst::getOpcodeForBitWidth(32)),
+ SrcAReg)
+ .add(LoadIdx.getIdxOp())
+ .add(LoadIdx.getOffsetOp());
+ LoadMIB.addMemOperand(*MI.memoperands_begin());
+ // Match what instruction selection does for a whole-dword access with a
+ // register index: record that the M0 write implied by the eventual movrel
+ // clobbers M0 (see AMDGPUAssignIdxToM0).
+ if (LoadIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Extract the accessed bits out of it.
+ MachineOperand BitOffset = MI.getOperand(4);
+ if (BitOffset.isImm())
+ BitOffset.setImm(BitOffset.getImm() & 31);
+
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ .addReg(SrcAReg)
+ .add(BitOffset)
+ .add(MI.getOperand(3)); // bitsize
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineFunction *MF = MBB->getParent();
+ auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB = BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ // The index is read again by the store below, so it does not die here.
+ LoadMIB->getOperand(1).setIsKill(false);
+ auto *StoreMMO = *MI.memoperands_begin();
+ // Synthesize a load MMO from the store's.
+ auto NewFlags = MachineMemOperand::MOLoad;
+ NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
+ LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ if (StoreIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Insert the stored bits into it.
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
+ .add(StoreIdx.getOperand(3)) // mask
+ .addReg(StoreIdx.getDataOp().getReg())
+ .addReg(SrcAReg);
+
+ // Write the dword back.
+ auto StoreMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_STORE_IDX_B32))
+ .addReg(DstAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ StoreMIB.addMemOperand(StoreMMO);
+ if (StoreIdx.getIdxOp().isReg())
+ StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+bool LowerIdxOps::run(MachineFunction &MF) {
+ bool Changed = false;
+
+ LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (auto MII = MBB.begin(), E = MBB.end(); MII != E;) {
+ MachineInstr &MI = *MII++;
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_BITS:
+ lowerLoadIdxBits(MI);
+ Changed = true;
+ break;
+ case AMDGPU::V_STORE_IDX_BITS:
+ lowerStoreIdxBits(MI);
+ Changed = true;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ return Changed;
+}
+
+class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return LowerIdxOps(MF).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Lower Idx Ops"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!LowerIdxOps(MF).run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPULowerIdxOpsLegacy::ID = 0;
+
+char &llvm::AMDGPULowerIdxOpsID = AMDGPULowerIdxOpsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPULowerIdxOpsLegacy, DEBUG_TYPE, "AMDGPU Lower Idx Ops",
+ false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
index 697c63e2079d1..970e9916f14dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -19,14 +19,19 @@ using namespace llvm;
using namespace AMDGPUMI;
unsigned VLoadStoreIdxInst::getBitWidth() const {
+ unsigned Opc = getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ llvm_unreachable("V_LOAD/STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
if (!Info)
llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
return Info->BitWidth;
}
int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_LOAD_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
if (!Info)
@@ -41,6 +46,8 @@ unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
}
int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
if (!Info)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index 24e24aab53152..e99594062871a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -37,7 +37,10 @@ class VLoadStoreIdxInst : public MachineInstr {
unsigned getBitWidth() const;
static bool classof(const MachineInstr *MI) {
- return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc) != nullptr;
}
};
@@ -47,8 +50,11 @@ class VLoadIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && !Info->IsStore;
}
};
@@ -59,8 +65,11 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && Info->IsStore;
}
};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d052f3c73920c..534d97df145c0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -115,6 +115,7 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
#endif
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-lower-idx-ops", AMDGPULowerIdxOpsPass())
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 6c426c6da11e8..3428047645339 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1387,6 +1387,12 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForGOpcs({G_AMDGPU_REG_STORE})
.Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD_BITS})
+ .Any({{BRC}, {{VgprBRC}, {Sgpr32_WF, Sgpr32, Vgpr32, Sgpr32}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE_BITS})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF, Vgpr32}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 5bdf0768179ec..e672498a75941 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3088,7 +3088,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// The dword index (operand 1) must be uniform; a divergent index needs a
// waterfall loop.
applyDefaultMapping(OpdMapper);
@@ -4495,10 +4497,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
break;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// data/result is a VGPR value; the dword index is uniform (SGPR).
OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ const unsigned Opc = MI.getOpcode();
+ if (Opc == AMDGPU::G_AMDGPU_REG_LOAD_BITS) {
+ // {dst, index, bitsize, bitoffset, is_sext_load}
+ OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+ OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+ } else if (Opc == AMDGPU::G_AMDGPU_REG_STORE_BITS) {
+ // {value, index, mask}
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ }
break;
}
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8c2732630e2e6..f296570ecce8c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -668,6 +668,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizePass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
+ initializeAMDGPULowerIdxOpsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1779,6 +1780,12 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert. The index of the access it expands has
+ // already been copied into M0 by SITargetLowering::finalizeLowering, and the
+ // whole-dword access created here inherits it.
+ addPass(&AMDGPULowerIdxOpsID);
+
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
}
@@ -2595,6 +2602,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert; see the comment in GCNPassConfig.
+ addMachineFunctionPass(AMDGPULowerIdxOpsPass(), PMW);
+
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
}
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b18ca6bd24040..5c9f8a2063847 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -90,6 +90,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUMemoryUtils.cpp
AMDGPUCoExecSchedStrategy.cpp
AMDGPUIGroupLP.cpp
+ AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1951195ae83f8..927227dadb317 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13206,17 +13206,17 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic (replacing the value with poison)
- // instead of failing instruction selection. Both callers - operation
- // legalization and the pre-ISel combine - replace the node with this result,
- // so the diagnostic is emitted exactly once.
+ // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
+ // with a diagnostic (replacing the value with poison) instead of failing
+ // instruction selection. Both callers - operation legalization and the
+ // pre-ISel combine - replace the node with this result, so the diagnostic is
+ // emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "whole-dword and 8-/16-bit loads and stores are implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13224,8 +13224,112 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
{DAG.getPOISON(Op.getValueType()), MemOp->getChain()}, DL);
};
- if (BitWidth < 32)
- return reportUnsupported();
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (BitWidth < 32) {
+ if (BitWidth != 8 && BitWidth != 16)
+ return reportUnsupported();
+
+ // Bail out for sub-dword types we cannot handle.
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
+ Load->getValueType(0).getSizeInBits() > 32)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore() &&
+ Store->getValue().getValueType() != MVT::i16 &&
+ Store->getValue().getValueType() != MVT::i32)
+ return reportUnsupported();
+ }
+
+ SDValue Ptr = MemOp->getBasePtr();
+
+ // Split the address into base and constant offset.
+ SDValue Base;
+ int32_t Offset;
+ if (DAG.isBaseWithConstantOffset(Ptr)) {
+ Base = Ptr.getOperand(0);
+ Offset = cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue();
+ } else if (auto *C = dyn_cast<ConstantSDNode>(Ptr)) {
+ Base = DAG.getConstant(0, DL, MVT::i32);
+ Offset = C->getSExtValue();
+ } else {
+ Base = Ptr;
+ Offset = 0;
+ }
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // constant.
+ KnownBits BaseKB;
+ if (Offset == 0 && MemOp->getAlign() >= Align(4)) {
+ BaseKB = KnownBits::makeConstant(APInt::getZero(2));
+ } else {
+ BaseKB = DAG.computeKnownBits(Base).trunc(2);
+ }
+
+ SDValue BitOffset;
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ BitOffset = DAG.getConstant((Offset & 3) * 8, DL, MVT::i32);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the least 5 bits of the bit
+ // offset, so we do not have to mask off the high bits.
+ BitOffset = DAG.getNode(ISD::SHL, DL, MVT::i32, Ptr,
+ DAG.getConstant(3, DL, MVT::i32));
+ }
+
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
+ DAG.getConstant(2, DL, MVT::i32));
+ SDValue BitSizeImm = DAG.getConstant(BitWidth, DL, MVT::i32);
+ SDValue Chain = MemOp->getChain();
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ EVT ValVT = Value.getValueType();
+ if (!ValVT.isScalarInteger())
+ Value = DAG.getNode(
+ ISD::BITCAST, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ValVT.getSizeInBits()), Value);
+ Value = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Value);
+
+ // ISD::SHL produces poison for shift amounts >= bit width, which generic
+ // DAG combines rely on. We need the mask for correctness, but it should
+ // get optimized away during final instruction selection.
+ SDValue ShiftOffset = DAG.getNode(ISD::AND, DL, MVT::i32, BitOffset,
+ DAG.getConstant(31, DL, MVT::i32));
+ Value = DAG.getNode(ISD::SHL, DL, MVT::i32, Value, ShiftOffset);
+
+ SDValue MaskBase = DAG.getConstant((1u << BitWidth) - 1, DL, MVT::i32);
+ SDValue Mask = DAG.getNode(ISD::SHL, DL, MVT::i32, MaskBase, ShiftOffset);
+
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE_BITS, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index, Mask}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ bool IsSExt = LoadOp->getExtensionType() == ISD::SEXTLOAD;
+ SDValue IsSExtImm = DAG.getConstant(IsSExt ? 1 : 0, DL, MVT::i32);
+ SDValue Value = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD_BITS, DL, DAG.getVTList(MVT::i32, MVT::Other),
+ {Chain, Index, BitSizeImm, BitOffset, IsSExtImm}, MemVT,
+ LoadOp->getMemOperand());
+ EVT ResVT = LoadOp->getValueType(0);
+ if (ResVT == MVT::i32)
+ return Value;
+
+ SDValue LoadChain = Value.getValue(1);
+ unsigned ResWidth = ResVT.getSizeInBits();
+ if (ResWidth < 32)
+ Value =
+ DAG.getNode(ISD::TRUNCATE, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ResWidth), Value);
+ if (Value.getValueType() != ResVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
+ return DAG.getMergeValues({Value, LoadChain}, DL);
+ }
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
@@ -18902,7 +19006,9 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
case AMDGPUISD::REG_LOAD:
- case AMDGPUISD::REG_STORE: {
+ case AMDGPUISD::REG_STORE:
+ case AMDGPUISD::REG_LOAD_BITS:
+ case AMDGPUISD::REG_STORE_BITS: {
const SIMachineFunctionInfo *MFI =
DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
unsigned NumAddressableVGPRs =
@@ -18910,7 +19016,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0;
+ unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
@@ -18918,10 +19024,22 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case AMDGPUISD::REG_STORE:
IndexOpIdx = 2;
break;
+ case AMDGPUISD::REG_LOAD_BITS:
+ IndexOpIdx = 1;
+ BitOffOpIdx = 3;
+ break;
+ case AMDGPUISD::REG_STORE_BITS:
+ IndexOpIdx = 2;
+ break;
}
if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
return SDValue(N, 0);
+ if (BitOffOpIdx) {
+ APInt BitOffMask = APInt::getLowBitsSet(32, 5);
+ if (SimplifyDemandedBits(N->getOperand(BitOffOpIdx), BitOffMask, DCI))
+ return SDValue(N, 0);
+ }
break;
}
case ISD::LOAD: {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6cb5b8b675331..ee0201ed2a523 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11019,7 +11019,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
// A VGPR ("as memory") indexed load is always divergent: it reads the wave's
// per-lane view of its vector registers, so even a uniform index yields a
// per-lane (divergent) value.
- if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD ||
+ Opcode == AMDGPU::G_AMDGPU_REG_LOAD_BITS)
return ValueUniformity::NeverUniform;
// Loads from the private and flat address spaces are divergent, because
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c6bbfc1ec14b4..0b72f2f0c021d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -70,6 +70,24 @@ def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+// VGPR address space sub-dword (8/16-bit) load/store with dword index. The
+// access is described by the dword index plus the bit position within that
+// dword; the store form carries a pre-shifted value and an insert mask.
+def SDTRegIdxBitsLoad : SDTypeProfile<1, 4,
+ [SDTCisInt<0>, // result (i32 or i16)
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>, // bitsize
+ SDTCisVT<3, i32>, // bitoffset
+ SDTCisVT<4, i32>]>; // is_sext_load
+def SDTRegIdxBitsStore : SDTypeProfile<0, 3,
+ [SDTCisInt<0>, // data (i32 or i16), pre-shifted insert value
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>]>; // insert mask
+def SIreg_load_bits : SDNode<"AMDGPUISD::REG_LOAD_BITS", SDTRegIdxBitsLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store_bits : SDNode<"AMDGPUISD::REG_STORE_BITS", SDTRegIdxBitsStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 1cc378611e212..0f77577d50035 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1161,6 +1161,70 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+// Used for byte or short loads.
+// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
+// sign-extended loads.
+//
+// Only the least 5 bits of $bitoffset are significant, the rest are ignored.
+// Only offset values of 0, 8, 16, or 24 are meaningful (after masking to the
+// 5 LSBs).
+//
+// Only bitsize of 8 or 16 bits are supported and must be a compile-time
+// constant.
+class V_LOAD_IDX_BITS<RegisterClass data_rc>
+ : VPseudoInstSI<(outs data_rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
+ VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
+
+// Used for byte or short stores.
+//
+// $data holds the pre-shifted insert value and $mask holds the insert mask
+// that is used when lowering to V_BFI_B32.
+class V_STORE_IDX_BITS<RegisterClass data_op_rc>
+ : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
+
+// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
+// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
+// shape folds a constant dword offset into the pseudo's $offset operand.
+multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
+ Instruction store_inst, ValueType vt> {
+ def : GCNPat<
+ (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+ def : GCNPat<
+ (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (store_inst $data, $idx, imm:$offset, $mask)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
+ (store_inst $data, $idx, 0, $mask)>;
+}
+
+defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4938,6 +5002,24 @@ def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
let mayStore = 1;
}
+// Sub-dword (8/16-bit) counterparts of the above. In addition to the dword
+// index they carry the bit position of the access within that dword, and are
+// lowered to a read-modify-write of the whole dword (see AMDGPULowerIdxOps).
+def G_AMDGPU_REG_LOAD_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index, type1:$bitsize,
+ type1:$bitoffset, type1:$is_sext_load);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index, type1:$mask);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
new file mode 100644
index 0000000000000..31c9b44e7236d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -0,0 +1,449 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
+; address space (13). A load reads the containing dword with an M0-relative
+; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
+; read-modify-write of that dword using v_bfi_b32 (see AMDGPULowerIdxOps).
+
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+define i32 @load_i8_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i8_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = sext i8 %x to i32
+ ret i32 %y
+}
+
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+define i32 @load_i16_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i16_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = sext i16 %x to i32
+ ret i32 %y
+}
+
+; A dword-aligned access has a statically known bit offset of zero.
+define i32 @load_i8_aligned(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+; A constant byte offset folds into a constant bit offset.
+define i32 @load_i8_const_offset(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i8_const_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 6
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_const_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 6
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 6
+ %x = load i8, ptr addrspace(13) %q, align 2
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+; GFX12-SDAG-LABEL: store_i16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-LABEL: store_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; A divergent index is made uniform by a waterfall loop.
+define i32 @load_i8_divergent(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 31, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-GISEL-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 75a744f49f10f..7c4e4b77a1eef 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,31 +1,36 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
-; not yet implemented. They must be rejected with a clean diagnostic on both
-; SelectionDAG and GlobalISel, rather than failing with "cannot select" /
-; "unable to legalize".
+; Accesses of the VGPR "as memory" address space (13) that are still not
+; implemented must be rejected with a clean diagnostic on both SelectionDAG and
+; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
+; Whole-dword and 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
+; as-vgpr-bits.ll.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; A sub-dword load extended into a value wider than a dword.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
- ret i8 %x
+ %y = zext i8 %x to i64
+ ret i64 %y
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define i16 @load_i16(ptr addrspace(13) inreg %p) {
- %x = load i16, ptr addrspace(13) %p
- ret i16 %x
+; A memory size that is neither a whole dword nor 8/16 bits.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define i1 @load_i1(ptr addrspace(13) inreg %p) {
+ %x = load i1, ptr addrspace(13) %p
+ ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
- store i8 %v, ptr addrspace(13) %p
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
+ store i1 %v, ptr addrspace(13) %p
ret void
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
- store i16 %v, ptr addrspace(13) %p
- ret void
+; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
+ %x = load <14 x i32>, ptr addrspace(13) %p
+ ret <14 x i32> %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 2f3bc3f259e89..f465ee4c0470c 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -69,6 +69,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -217,6 +218,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-lower-idx-ops
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -405,6 +407,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-lower-idx-ops
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index d10df2ed1d11f..1fd95e2289480 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -351,6 +352,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -673,6 +675,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -999,6 +1002,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Lower Idx Ops
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1339,6 +1343,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Lower Idx Ops
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From 9e67b142e1ff9a237cc680366d7db56e5773b4b2 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:14:32 +0300
Subject: [PATCH 02/19] Correctly diagnose the mis-aligned sub-dowrd accesses
instead of miscompiling them
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 27 ++++++++----
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 19 +++++---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 43 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 33 ++++++++++----
4 files changed, 99 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c23e510dab4c3..381d433a2a147 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -557,11 +557,21 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
-// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
-// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
-static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+// memory access producing/consuming a ValSize-bit value at the given alignment.
+// Whole-dword accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo)
+// are supported, as are 8-/16-bit accesses, including extending loads into a
+// 16- or 32-bit value.
+//
+// A sub-dword access is implemented as a bit-field extract from (or insert
+// into) the dword containing it, so it must not straddle a dword boundary. An
+// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
+// natural alignment covers both, and is what lets the bit offset within the
+// dword be computed from a possibly dynamic pointer.
+static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment) {
if (MemSize == 8 || MemSize == 16) {
+ if (Alignment < Align(MemSize / 8))
+ return false;
if (ValSize == MemSize)
return true;
if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
@@ -3530,14 +3540,15 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
- // with a diagnostic instead of failing to legalize.
- if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize)) {
+ // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
+ // Reject anything else with a diagnostic instead of failing to legalize.
+ if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and 8-/16-bit loads and stores are implemented",
+ "whole-dword and naturally aligned 8-/16-bit loads and stores are "
+ "implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 927227dadb317..0ba68de402ddb 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13206,17 +13206,18 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
- // with a diagnostic (replacing the value with poison) instead of failing
- // instruction selection. Both callers - operation legalization and the
- // pre-ISel combine - replace the node with this result, so the diagnostic is
- // emitted exactly once.
+ // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
+ // Reject anything else with a diagnostic (replacing the value with poison)
+ // instead of failing instruction selection. Both callers - operation
+ // legalization and the pre-ISel combine - replace the node with this result,
+ // so the diagnostic is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and 8-/16-bit loads and stores are implemented",
+ "whole-dword and naturally aligned 8-/16-bit loads and stores are "
+ "implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13231,6 +13232,12 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
if (BitWidth != 8 && BitWidth != 16)
return reportUnsupported();
+ // The access becomes a bit-field extract from (or insert into) the dword
+ // containing it, so it must not straddle a dword boundary. An 8-bit access
+ // never can; a 16-bit one only if it is 2-byte aligned.
+ if (MemOp->getAlign() < Align(BitWidth / 8))
+ return reportUnsupported();
+
// Bail out for sub-dword types we cannot handle.
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 31c9b44e7236d..4b1703c1b6043 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -447,3 +447,46 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
store i8 %v, ptr addrspace(13) %p
ret void
}
+
+; A 2-byte aligned 16-bit access sits at bit offset 0 or 16, so it always fits
+; within the containing dword even though the offset is not known statically.
+; (An under-aligned 16-bit access is rejected; see as-vgpr-unsupported.ll.)
+define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i16_offset2:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i16_offset2:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 2
+ %x = load i16, ptr addrspace(13) %q, align 2
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 7c4e4b77a1eef..6a8c7772716a1 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,14 +1,29 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are still not
-; implemented must be rejected with a clean diagnostic on both SelectionDAG and
-; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
-; Whole-dword and 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
-; as-vgpr-bits.ll.
+; Accesses of the VGPR "as memory" address space (13) that are not implemented
+; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
+; rather than failing with "cannot select" / "unable to legalize" - or, worse,
+; silently generating wrong code. Whole-dword and naturally aligned 8-/16-bit
+; accesses are implemented; see as-vgpr-basic.ll and as-vgpr-bits.ll.
+
+; An under-aligned 16-bit access may straddle a dword boundary, which the
+; bit-field extract / insert lowering cannot express.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i16_align1(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p, align 1
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i16_align1(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
; A sub-dword load extended into a value wider than a dword.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
%y = zext i8 %x to i64
@@ -16,20 +31,20 @@ define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
}
; A memory size that is neither a whole dword nor 8/16 bits.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i1 @load_i1(ptr addrspace(13) inreg %p) {
%x = load i1, ptr addrspace(13) %p
ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
store i1 %v, ptr addrspace(13) %p
ret void
}
; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
%x = load <14 x i32>, ptr addrspace(13) %p
ret <14 x i32> %x
>From 47983b602c342a8b3771abef70813bb8bb6915b0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:12:39 +0300
Subject: [PATCH 03/19] Run AMDGPULowerIdxOps for optnone functions in
AMDGPULowerIdxOps
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 5 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 110 ++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 95cfe42df7e6d..e7d3a1e8bf172 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -168,8 +168,9 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: nothing else expands the
+ // sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
+ // therefore must not be skipped for optnone functions.
return LowerIdxOps(MF).run(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4b1703c1b6043..66790a4a155fe 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -490,3 +490,113 @@ define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
%y = zext i16 %x to i32
ret i32 %y
}
+
+; Expanding these pseudos is required lowering, so it has to happen for optnone
+; functions too - clang marks every function optnone at -O0. Nothing else can
+; lower them, so if the expansion is skipped AMDGPULowerVGPREncoding is left
+; with a pseudo it cannot handle.
+define i32 @load_i8_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-SDAG-LABEL: load_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_and_b32_e64 v0, v0, s0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnone {
+; GFX12-SDAG-LABEL: store_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ; kill: def $vgpr1_lo16 killed $vgpr0 killed $exec
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
>From 576e2a05206d67e815f68f79a7979735b723fdb7 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:50:21 +0300
Subject: [PATCH 04/19] Add a MIR test for AMDGPULowerIdxOps
---
.../as-vgpr-lower-idx-ops.mir | 134 ++++++++++++++++++
1 file changed, 134 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
new file mode 100644
index 0000000000000..c991990db933e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+
+# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
+# AMDGPULowerIdxOps, in isolation:
+#
+# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+#
+# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
+# virtual here and the implicit-def $m0 that instruction selection attached is
+# carried over to the whole-dword accesses the expansion creates.
+
+---
+name: lower_load_idx_bits_zext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_zext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_sext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_sext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 1, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_16_offset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_16_offset
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 3, 16, 16, 0, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+# A dynamic bit offset is passed straight through to the bit-field extract;
+# only its low 5 bits are significant.
+---
+name: lower_load_idx_bits_dynamic_bitoffset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_dynamic_bitoffset
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, %1, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2
+...
+
+# The store is a read-modify-write of the containing dword. The index is read
+# again by the store, so it must not be killed by the load.
+---
+name: lower_store_idx_bits
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_store_idx_bits
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ V_STORE_IDX_BITS %1, killed %0, 0, 255, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ S_ENDPGM 0
+...
+
+---
+name: lower_store_idx_bits_dynamic_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: lower_store_idx_bits_dynamic_mask
+ ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = COPY $vgpr1
+ V_STORE_IDX_BITS %1, %0, 2, %2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ S_ENDPGM 0
+...
>From 5290726f03f21aa13c1a411788892f4932d59a56 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:51:08 +0300
Subject: [PATCH 05/19] Verify operand invariants of the VGPR-memory indexed
accesses
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 28 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 26 +++++++++++++++++
2 files changed, 54 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ee0201ed2a523..3d659db618cfb 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5225,6 +5225,34 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
+ // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
+ // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
+ // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
+ // access has to record that the move clobbers M0, otherwise it could be
+ // separated from the code that sets M0 up.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (MI.getNumMemOperands() != 1) {
+ ErrInfo = "v_load/store_idx should have exactly one memory operand.";
+ return false;
+ }
+
+ const MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg()) {
+ if (IdxOp.getSubReg() != 0) {
+ ErrInfo =
+ "v_load/store_idx register index must not have a subregister.";
+ return false;
+ }
+
+ if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
+ !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
+ ErrInfo = "v_load/store_idx with register index must have implicit-def "
+ "$m0 on movrel subtargets.";
+ return false;
+ }
+ }
+ }
+
// Make sure the register classes are correct.
for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
new file mode 100644
index 0000000000000..bec5ef54aa203
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -0,0 +1,26 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
+# access ends up in M0, which the v_movrel[sd] emitted by
+# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
+# index to M0, the access must record that the eventual move clobbers M0 -
+# otherwise it could be separated from the code that sets M0 up. Check that the
+# machine verifier rejects a register index without that implicit-def, for both
+# the whole-dword and the sub-dword pseudos.
+
+# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
+
+---
+name: missing_m0_implicit_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
+ %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From f47aab7406f04f091c290393f4506c8e780ee0e7 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:59:26 +0300
Subject: [PATCH 06/19] Test the bit-offset half of the VGPR-memory
demanded-bits combine
---
.../as-vgpr-index-demanded-bits.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index e1dfb50df7fb9..ef92bd4fbdfb1 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -7,6 +7,11 @@
; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
; path the mask must not survive into the M0 index computation.
+;
+; The same combine also trims the bit offset of a sub-dword access, which needs
+; only its low 5 bits to name a bit position within the containing dword. That
+; half matters independently: a mask can be dead for the bit offset while the
+; index computation keeps it alive, and vice versa.
define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-SDAG-LABEL: load_masked_index:
@@ -47,5 +52,79 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
store i32 %val, ptr addrspace(13) %ptr
ret void
}
+
+; A dynamic pointer masked with 0x3ff: the mask is dead both for the index and
+; for the bit offset, so no s_and may remain.
+define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %m = and i32 %sum, 1023
+ %ptr = inttoptr i32 %m to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
+
+; Setting a high bit of the pointer cannot affect the 5-bit bit offset either,
+; so no bit-set of the shifted offset may remain.
+define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_or_high_bit:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_or_high_bit:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %o = or i32 %sum, 65536
+ %ptr = inttoptr i32 %o to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
>From 61b7d1bfa7e8e35ae2b99ba935404516fcc398b4 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:13:46 +0300
Subject: [PATCH 07/19] Simplify the sub-dword VGPR-memory pseudo definitions
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 57 +++++++++++-------------
1 file changed, 25 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 0f77577d50035..2154551a889e8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1171,10 +1171,12 @@ defm : VRegIdxLoadStorePat<vt>;
//
// Only bitsize of 8 or 16 bits are supported and must be a compile-time
// constant.
-class V_LOAD_IDX_BITS<RegisterClass data_rc>
- : VPseudoInstSI<(outs data_rc:$data),
- (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
- VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+// The lowering normalizes the loaded value to a dword, so only a 32-bit $data
+// form is needed.
+def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
+ (ins SReg_32:$idx, i32imm:$offset,
+ i32imm:$bitsize, VSrc_b32:$bitoffset,
+ i32imm:$is_sext_load)> {
let mayLoad = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1182,15 +1184,13 @@ class V_LOAD_IDX_BITS<RegisterClass data_rc>
let Defs = [M0];
}
-def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
-
// Used for byte or short stores.
//
// $data holds the pre-shifted insert value and $mask holds the insert mask
// that is used when lowering to V_BFI_B32.
-class V_STORE_IDX_BITS<RegisterClass data_op_rc>
- : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
- i32imm:$offset, VSrc_b32:$mask)> {
+def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
+ (ins VGPR_32:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1198,32 +1198,25 @@ class V_STORE_IDX_BITS<RegisterClass data_op_rc>
let Defs = [M0];
}
-def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
-
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
// shape folds a constant dword offset into the pseudo's $offset operand.
-multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
- Instruction store_inst, ValueType vt> {
- def : GCNPat<
- (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
- (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
- imm:$is_sext_load)>;
- def : GCNPat<
- (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
- (store_inst $data, $idx, imm:$offset, $mask)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
- (store_inst $data, $idx, 0, $mask)>;
-}
-
-defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+def : GCNPat<
+ (i32 (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+def : GCNPat<
+ (i32 (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, imm:$offset, $mask)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, i32:$idx, i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, 0, $mask)>;
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
>From 26987bc942c35afb4720c076d3cefc5004204f10 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:24:43 +0300
Subject: [PATCH 08/19] Cover the remaining VGPR-memory operand invariants
---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 38 +++++++++++++------
1 file changed, 26 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
index bec5ef54aa203..db32b6957abb3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -1,26 +1,40 @@
# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
-# access ends up in M0, which the v_movrel[sd] emitted by
-# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
-# index to M0, the access must record that the eventual move clobbers M0 -
-# otherwise it could be separated from the code that sets M0 up. Check that the
-# machine verifier rejects a register index without that implicit-def, for both
-# the whole-dword and the sub-dword pseudos.
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
+# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
+# rewritten the index to M0, the access must record that the eventual move
+# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
+# The lowering also relies on the access having exactly one memory operand (it
+# synthesizes a load operand from it for a sub-dword store), and on the index
+# carrying no subregister, since it is rewritten in place to M0.
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
---
-name: missing_m0_implicit_def
+name: operand_invariants
tracksRegLiveness: true
body: |
bb.0:
- liveins: $sgpr0, $vgpr0
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
- %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
+
+ ; Missing implicit-def $m0, for each of the four pseudo forms.
+ %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %2, implicit %3
+ S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
...
>From 3c269fc031cee89552c82631e4247c57fb3bde06 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:07:38 +0300
Subject: [PATCH 09/19] Also build the sub-dword VGPR-memory tests for wave64
---
llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 66790a4a155fe..992b737009947 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,6 +2,14 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
+; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
+; sub-dword code is the same there modulo register allocation, so only compile
+; and verify rather than checking the output a second time. What differs is the
+; waterfall around a divergent access, which uses a 64-bit exec mask.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
>From 0718035565dd1b65ab5b086401581fb88bd8f55c Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:28:47 +0300
Subject: [PATCH 10/19] Test sub-dword VGPR-memory accesses across control flow
---
.../as-vgpr-bits-control-flow.ll | 316 ++++++++++++++++++
1 file changed, 316 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
new file mode 100644
index 0000000000000..73edbbb7855e5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
+; across control flow. A sub-dword store is a read-modify-write of the dword
+; containing it, which makes it sensitive to how it is placed relative to other
+; accesses of the same dword, to loop bodies, and to the exec mask.
+
+; Two stores into different bytes of the same dword. Each is a full
+; read-modify-write, so the second has to observe the first: if the two dword
+; loads were merged, the first byte would be lost.
+define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
+; GFX12-SDAG-LABEL: two_bytes_same_dword:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: two_bytes_same_dword:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 1
+ store i8 %a, ptr addrspace(13) %p
+ store i8 %b, ptr addrspace(13) %q
+ ret void
+}
+
+; A store inside a loop. The pointer deliberately depends on the induction
+; variable: with a loop-invariant pointer the whole access is hoisted out and
+; nothing inside the loop is exercised. M0 has to be re-established for each
+; access on every iteration.
+define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
+; GFX12-SDAG-LABEL: store_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ store i8 %v, ptr addrspace(13) %q
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Likewise for a load, accumulating so the result cannot be folded away.
+define i32 @load_in_loop(ptr addrspace(13) inreg %p, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: .LBB2_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: .LBB2_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ %x = load i8, ptr addrspace(13) %q
+ %z = zext i8 %x to i32
+ %acc.next = add i32 %acc, %z
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; A store under divergent control flow: the read-modify-write must stay inside
+; the conditional region so that lanes which did not take the branch keep their
+; dword unmodified.
+define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
+; GFX12-SDAG-LABEL: store_divergent_if:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
+; GFX12-SDAG-NEXT: ; %bb.1: ; %then
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_divergent_if:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-GISEL-NEXT: ; %bb.1: ; %then
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %c = icmp ult i32 %tid, 8
+ br i1 %c, label %then, label %exit
+
+then:
+ store i8 %v, ptr addrspace(13) %p
+ br label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From 089396350cc3a8173f6c55671b19346de2456cd3 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:59:30 +0300
Subject: [PATCH 11/19] NFC: track the downstream implementation more closely
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 35 ++++++++++++--------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
2 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index e7d3a1e8bf172..815605099cd90 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -36,9 +36,9 @@ using namespace llvm;
namespace {
-class LowerIdxOps {
+class AMDGPULowerIdxOpsImpl {
public:
- LowerIdxOps(MachineFunction &MF)
+ AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
MRI(&MF.getRegInfo()) {}
@@ -52,7 +52,7 @@ class LowerIdxOps {
MachineRegisterInfo *MRI;
};
-void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -69,19 +69,21 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
- LoadMIB.addMemOperand(*MI.memoperands_begin());
+ auto *LoadMMO = *MI.memoperands_begin();
+ LoadMIB.addMemOperand(LoadMMO);
// Match what instruction selection does for a whole-dword access with a
// register index: record that the M0 write implied by the eventual movrel
// clobbers M0 (see AMDGPUAssignIdxToM0).
if (LoadIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+ Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
MachineOperand BitOffset = MI.getOperand(4);
if (BitOffset.isImm())
BitOffset.setImm(BitOffset.getImm() & 31);
- BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, DataReg)
.addReg(SrcAReg)
.add(BitOffset)
.add(MI.getOperand(3)); // bitsize
@@ -91,11 +93,15 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent();
auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+ const MCInstrDesc &II = TII->get(AMDGPU::V_BFI_B32_e64);
+
+ Register DataReg = StoreIdx.getDataOp().getReg();
+
Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
@@ -110,15 +116,16 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
// Synthesize a load MMO from the store's.
auto NewFlags = MachineMemOperand::MOLoad;
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
- LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
+ LoadMIB.addMemOperand(LoadMMO);
if (StoreIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
- BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
- .add(StoreIdx.getOperand(3)) // mask
- .addReg(StoreIdx.getDataOp().getReg())
- .addReg(SrcAReg);
+ auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
+ CoreMIB.add(StoreIdx.getOperand(3)); // mask
+ CoreMIB.addReg(DataReg);
+ CoreMIB.addReg(SrcAReg);
// Write the dword back.
auto StoreMIB =
@@ -135,7 +142,7 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-bool LowerIdxOps::run(MachineFunction &MF) {
+bool AMDGPULowerIdxOpsImpl::run(MachineFunction &MF) {
bool Changed = false;
LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
@@ -171,7 +178,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
// This is required lowering, not an optimization: nothing else expands the
// sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
// therefore must not be skipped for optnone functions.
- return LowerIdxOps(MF).run(MF);
+ return AMDGPULowerIdxOpsImpl(MF).run(MF);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -187,7 +194,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
PreservedAnalyses
AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!LowerIdxOps(MF).run(MF))
+ if (!AMDGPULowerIdxOpsImpl(MF).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0ba68de402ddb..9f334be564f7f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19023,7 +19023,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
+ unsigned IndexOpIdx, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
>From cfa33c26c83c4c7d3005bf1359b1154b2a7de1b8 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:16:13 +0300
Subject: [PATCH 12/19] Do not describe sub-dword VGPR-memory accesses by index
and width
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3d659db618cfb..e14de0e34fb49 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -551,6 +551,13 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
}
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ // A sub-dword access is not described by a dword index and width alone -
+ // the bit position within the dword is part of the address - and has no
+ // entry in the width table. Report it as opaque so that callers such as
+ // areMemAccessesTriviallyDisjoint() fall back to assuming an overlap.
+ if (!AMDGPU::getVLdStIdxOpcodeInfoByOpcode(LdSt.getOpcode()))
+ return false;
+
BaseOp = &LdStIdx->getIdxOp();
OffsetOp = &LdStIdx->getOffsetOp();
>From f5466d40f8f6ff53b5d85fb80dc0a7f3bfc4faab Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:47:57 +0300
Subject: [PATCH 13/19] Fix tests
---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 15 ++-
.../as-vgpr-index-demanded-bits.ll | 105 ++++++++++++++++++
2 files changed, 115 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 992b737009947..4910e9c56c4b7 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,13 +2,18 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
-; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
-; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
-; sub-dword code is the same there modulo register allocation, so only compile
-; and verify rather than checking the output a second time. What differs is the
-; waterfall around a divergent access, which uses a 64-bit exec mask.
+; Also compile for two further movrel subtargets. The expansion itself is
+; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
+; the sub-dword code is the same on both modulo register allocation, so only
+; compile and verify rather than checking the output again. gfx942 differs in
+; using a 64-bit exec mask for the waterfall around a divergent access, and
+; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
+; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
+; functions also cover the S_SET_VGPR_MSB encoding.
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index ef92bd4fbdfb1..4948db66c3023 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
@@ -29,6 +31,28 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
%v = load i32, ptr addrspace(13) %ptr
@@ -47,6 +71,24 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-LABEL: store_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: store_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_endpgm
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
store i32 %val, ptr addrspace(13) %ptr
@@ -82,6 +124,37 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%m = and i32 %sum, 1023
%ptr = inttoptr i32 %m to ptr addrspace(13)
@@ -119,6 +192,37 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%o = or i32 %sum, 65536
%ptr = inttoptr i32 %o to ptr addrspace(13)
@@ -128,3 +232,4 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
+; GFX1250: {{.*}}
>From c761f131fb1e24444ad5036aedfc3c61e58a0126 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 18:14:45 +0300
Subject: [PATCH 14/19] Check the encodings of the sub-dword VGPR-memory
accesses
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 12 +---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 26 +++----
.../as-vgpr-bits-control-flow.ll | 7 +-
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 71 +++++++++++++++++++
.../as-vgpr-lower-idx-ops.mir | 16 ++---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 40 -----------
.../AddressSpaceVGPR/as-vgpr-verifier.mir | 32 +++++++++
7 files changed, 125 insertions(+), 79 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 815605099cd90..ec1463e0b2eb2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -15,7 +15,8 @@
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup, and before register allocation
+/// here take part in the usual M0 setup - they declare their write of M0
+/// themselves, like the pseudo they replace - and before register allocation
/// because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -71,11 +72,6 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
.add(LoadIdx.getOffsetOp());
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
- // Match what instruction selection does for a whole-dword access with a
- // register index: record that the M0 write implied by the eventual movrel
- // clobbers M0 (see AMDGPUAssignIdxToM0).
- if (LoadIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
@@ -118,8 +114,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
LoadMIB.addMemOperand(LoadMMO);
- if (StoreIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
@@ -134,8 +128,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
StoreMIB.addMemOperand(StoreMMO);
- if (StoreIdx.getIdxOp().isReg())
- StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e14de0e34fb49..ebc4d6f2e379b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5232,11 +5232,11 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
- // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
- // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
- // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
- // access has to record that the move clobbers M0, otherwise it could be
- // separated from the code that sets M0 up.
+ // The lowering of a VGPR "as memory" indexed access relies on it having
+ // exactly one memory operand, since a sub-dword store synthesizes a load
+ // operand from it, and on the index carrying no subregister, since
+ // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
+ // is declared by the instruction itself, so it needs no check here.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
@@ -5244,19 +5244,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
const MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg()) {
- if (IdxOp.getSubReg() != 0) {
- ErrInfo =
- "v_load/store_idx register index must not have a subregister.";
- return false;
- }
-
- if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
- !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
- ErrInfo = "v_load/store_idx with register index must have implicit-def "
- "$m0 on movrel subtargets.";
- return false;
- }
+ if (IdxOp.isReg() && IdxOp.getSubReg() != 0) {
+ ErrInfo = "v_load/store_idx register index must not have a subregister.";
+ return false;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 73edbbb7855e5..1f20bf28b2335 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -37,15 +37,16 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
new file mode 100644
index 0000000000000..32921a6244ebe
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -0,0 +1,71 @@
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+
+; The sub-dword accesses of the VGPR "as memory" address space (13) that
+; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
+; check assembly text, which does not show that the operand forms the expansion
+; picks can be assembled and encoded: a mask it builds from a known bit offset is
+; too wide for an inline constant and becomes a 32-bit literal, and a bit offset
+; reaches the bit-field extract as either an inline constant or an SGPR. On a
+; subtarget with more than 256 addressable VGPRs the whole-dword accesses the
+; expansion creates additionally carry the S_SET_VGPR_MSB encoding, so check
+; there too.
+;
+; The expansion runs after instruction selection and is handed the same pseudos
+; by both selectors, which produce identical code for these functions, so one
+; selector is enough here. as-vgpr-bits.ll covers the two separately.
+
+; A statically known bit offset turns the mask into a literal operand of the
+; bit-field insert.
+define amdgpu_ps void @store_i8_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_aligned>:
+; GFX1250: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i16_aligned>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xffff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i16
+ store i16 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; Without one the mask is computed at run time and is an SGPR instead.
+define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_dynamic_mask>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; Both bit offset and width of the extract are inline constants here.
+define amdgpu_ps i32 @load_i8_zext_aligned(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_zext_aligned>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_u32 v0, v0, 0, 8
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %z = zext i8 %x to i32
+ ret i32 %z
+}
+
+; A run-time bit offset is an SGPR, and the sign-extending load selects the
+; signed extract.
+define amdgpu_ps i32 @load_i8_sext_dynamic(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_sext_dynamic>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_i32 v0, v0, s0, 8
+ %x = load i8, ptr addrspace(13) %p
+ %s = sext i8 %x to i32
+ ret i32 %s
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c991990db933e..c8b4fefa3c3e3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -21,7 +21,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +39,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +57,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +78,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +100,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +122,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
deleted file mode 100644
index db32b6957abb3..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ /dev/null
@@ -1,40 +0,0 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-
-# Operand invariants of the VGPR "as memory" (address space 13) indexed
-# accesses, as enforced by SIInstrInfo::verifyInstruction.
-#
-# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
-# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
-# rewritten the index to M0, the access must record that the eventual move
-# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
-# The lowering also relies on the access having exactly one memory operand (it
-# synthesizes a load operand from it for a sub-dword store), and on the index
-# carrying no subregister, since it is rewritten in place to M0.
-
-# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
-# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
-# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
-
----
-name: operand_invariants
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
- %0:sreg_32 = COPY $sgpr0
- %1:vgpr_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr2_sgpr3
-
- ; No memory operand.
- %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
-
- ; Index with a subregister.
- %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
-
- ; Missing implicit-def $m0, for each of the four pseudo forms.
- %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
- V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
- V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
-...
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
new file mode 100644
index 0000000000000..d4e7ec49e1f40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -0,0 +1,32 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# The lowering relies on the access having exactly one memory operand, since a
+# sub-dword store synthesizes a load operand from it, and on the index carrying
+# no subregister, since AMDGPUAssignIdxToM0 rewrites it in place to M0. That the
+# access writes M0 is declared by the instructions themselves, so it needs no
+# check: MIR without the operand does not parse.
+
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
+
+---
+name: operand_invariants
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit-def $m0, implicit $exec
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit-def $m0, implicit $exec :: (load (s32), addrspace 13)
+
+ S_ENDPGM 0, implicit %3, implicit %4
+...
>From 98970366fb0f7f4e59a658b9976a400113e4a362 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 31 Jul 2026 20:15:02 +0300
Subject: [PATCH 15/19] Document the sub-dword access rules for the VGPR
address space
---
llvm/docs/AMDGPUUsage.rst | 7 +++++
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 26 ++++++++++---------
2 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index bacc70d98b5cc..4630104453271 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1345,6 +1345,13 @@ supported for the ``amdgcn`` target.
of ``v3``. Storing 8 bytes to address ``32`` overwrites the contents of
``v[8:9]``.
+ An access narrower than a dword is implemented by extracting or inserting the
+ bits within the dword that contains it, so it must not straddle a dword
+ boundary: 8-bit accesses are unrestricted, while 16-bit accesses must be
+ 2-byte aligned. Wider accesses must be a whole number of dwords. Anything
+ else, such as an under-aligned 16-bit access, is reported as an error rather
+ than miscompiled.
+
Use of this address space by frontends is strongly discouraged. It has unusual
and subtle lifetime rules due to the potential for interaction with normal
register allocation. It exists primarily for internal purposes of the backend,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 381d433a2a147..0f88afbf20e37 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1835,7 +1835,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
// unsupported access is diagnosed cleanly rather than failing to
// legalize.
- .customIf([=](const LegalityQuery &Query) -> bool {
+ .customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3568,7 +3568,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register BaseReg = PtrReg;
int64_t Offset = 0;
if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
- if (auto MaybeOff =
+ if (std::optional<ValueAndVReg> MaybeOff =
getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
BaseReg = PtrAdd->getBaseReg();
Offset = MaybeOff->Value.getSExtValue();
@@ -3581,7 +3581,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
HaveConstantBitOffset = true;
ConstantBitOffsetVal = 0;
} else {
- auto &VT = *Helper.getValueTracking();
+ GISelValueTracking &VT = *Helper.getValueTracking();
KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
if (BaseKB.isConstant()) {
Offset += BaseKB.getConstant().getZExtValue();
@@ -3591,18 +3591,18 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
}
// Setup common registers.
- const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
- auto Two = B.buildConstant(I32, 2);
- const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+ const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ MachineInstrBuilder Two = B.buildConstant(I32, 2);
+ const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
- const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ const MachineInstrBuilder BitWidthReg = B.buildConstant(I32, MemSize);
Register BitOffsetReg;
if (HaveConstantBitOffset) {
BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
} else {
// V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
// offset, so masking the high bits is unnecessary.
- auto Three = B.buildConstant(I32, 3);
+ MachineInstrBuilder Three = B.buildConstant(I32, 3);
BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
}
@@ -3616,19 +3616,21 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register ShiftAmt = BitOffsetReg;
if (!HaveConstantBitOffset) {
- auto Mask = B.buildConstant(I32, 31);
+ MachineInstrBuilder Mask = B.buildConstant(I32, 31);
ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
}
Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
- const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
- const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+ const MachineInstrBuilder MaskBase =
+ B.buildConstant(I32, (1u << MemSize) - 1);
+ const MachineInstrBuilder Mask = B.buildShl(I32, MaskBase, ShiftAmt);
B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
.addMemOperand(&MMO);
} else {
const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
- const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ const MachineInstrBuilder IsSExtReg =
+ B.buildConstant(I32, IsSExt ? 1 : 0);
Register Result =
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
{Index, BitWidthReg, BitOffsetReg, IsSExtReg})
>From 1653e93d9ff04ed1bf0662bdc4ea5fd3b22d11d6 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 3 Aug 2026 20:04:45 +0300
Subject: [PATCH 16/19] Adopt subarch triples in the sub-dword tests and drop
the redundant VALU
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../AddressSpaceVGPR/as-vgpr-bits-control-flow.ll | 8 ++++----
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 4 ++--
.../CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 12 ++++++------
.../AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir | 2 +-
6 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 2154551a889e8..70f872c6d0662 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1178,7 +1178,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
i32imm:$bitsize, VSrc_b32:$bitoffset,
i32imm:$is_sext_load)> {
let mayLoad = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1192,7 +1191,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
(ins VGPR_32:$data, SReg_32:$idx,
i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 1f20bf28b2335..23bb00886d179 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
; across control flow. A sub-dword store is a read-modify-write of the dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 32921a6244ebe..504e8034658cf 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.00-- -o - %s | llvm-mc -triple=amdgpu12.00-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -mtriple=amdgpu12.50-- -o - %s | llvm-mc -triple=amdgpu12.50-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
; The sub-dword accesses of the VGPR "as memory" address space (13) that
; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4910e9c56c4b7..9e7bea5d5f8cd 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for two further movrel subtargets. The expansion itself is
; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
@@ -10,10 +10,10 @@
; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
; functions also cover the S_SET_VGPR_MSB encoding.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c8b4fefa3c3e3..03646b899cbae 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.00-- -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
# AMDGPULowerIdxOps, in isolation:
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
index d4e7ec49e1f40..f42d75a904bfc 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgpu12.00-- -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
# Operand invariants of the VGPR "as memory" (address space 13) indexed
# accesses, as enforced by SIInstrInfo::verifyInstruction.
>From 8238d1c4e28ba29e8fcc987cc94a685d572f1860 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 15:22:07 +0300
Subject: [PATCH 17/19] Regenerate the sub-dword expectations for the M0 setup
change
---
.../as-vgpr-bits-control-flow.ll | 88 ++++++++-----------
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 79 +++++++----------
3 files changed, 67 insertions(+), 102 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 23bb00886d179..51f5e1fb468ff 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -22,31 +22,27 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -58,32 +54,29 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -107,10 +100,9 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
@@ -119,7 +111,6 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
@@ -136,20 +127,18 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, s3, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v2, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
@@ -257,16 +246,14 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
; GFX12-SDAG-NEXT: ; %bb.1: ; %then
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
@@ -285,19 +272,16 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
; GFX12-GISEL-NEXT: ; %bb.1: ; %then
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 504e8034658cf..5638ce53c394b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -42,7 +42,7 @@ define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %
define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
; DIS-LABEL: <store_i8_dynamic_mask>:
; DIS: v_movrels_b32_e32 v1, v0
-; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_bfi_b32 v0, s0, v0, v1
; DIS: v_movreld_b32_e32 v0, v0
%v = trunc i32 %vv to i8
store i8 %v, ptr addrspace(13) %p
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 9e7bea5d5f8cd..1c3f1db234aec 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -207,16 +207,14 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -229,19 +227,16 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
@@ -257,16 +252,14 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -279,19 +272,16 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i16 %v, ptr addrspace(13) %p
@@ -306,11 +296,8 @@ define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s0, s0, 2
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-NEXT: s_mov_b32 m0, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
; GFX12-NEXT: v_movreld_b32_e32 v0, v0
@@ -403,11 +390,10 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
@@ -444,7 +430,6 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
@@ -568,15 +553,13 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, s1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -589,25 +572,23 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 31
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_and_b32 s1, s0, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 0xff
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s1, s0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
>From 99a5c25a3668de4151401246646f82e3178753f7 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 17:43:50 +0300
Subject: [PATCH 18/19] Inherit the M0 operand when expanding the sub-dword
VGPR-memory pseudos
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 25 ++++++++++++++++---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +--
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../as-vgpr-lower-idx-ops.mir | 23 +++++++++--------
4 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index ec1463e0b2eb2..ba632ce02074c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -14,10 +14,9 @@
/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
-/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup - they declare their write of M0
-/// themselves, like the pseudo they replace - and before register allocation
-/// because it introduces new virtual registers.
+/// This runs after SITargetLowering::finalizeLowering has given the pseudo its
+/// M0 operand, so the whole-dword accesses created here inherit it, and before
+/// register allocation because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -41,6 +40,7 @@ class AMDGPULowerIdxOpsImpl {
public:
AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ TRI(MF.getSubtarget<GCNSubtarget>().getRegisterInfo()),
MRI(&MF.getRegInfo()) {}
bool run(MachineFunction &MF);
@@ -48,11 +48,25 @@ class AMDGPULowerIdxOpsImpl {
private:
void lowerLoadIdxBits(MachineInstr &MI);
void lowerStoreIdxBits(MachineInstr &MI);
+ void inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const;
const SIInstrInfo *TII;
+ const SIRegisterInfo *TRI;
MachineRegisterInfo *MRI;
};
+// An indexed access takes its index from M0 where the subtarget has movrel, and
+// clobbers M0 through s_set_gpr_idx_on where it indexes with the VGPR indexing
+// mode. SITargetLowering::finalizeLowering picks between the two and records a
+// clobber as an implicit def, so a whole-dword access replacing the pseudo here
+// has to carry that over: the index itself comes along with the operand.
+void AMDGPULowerIdxOpsImpl::inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const {
+ if (MI.definesRegister(AMDGPU::M0, TRI))
+ MIB.addDef(AMDGPU::M0, RegState::Implicit);
+}
+
void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -70,6 +84,7 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
Register DataReg = LoadIdx.getDataOp().getReg();
@@ -106,6 +121,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
// The index is read again by the store below, so it does not die here.
LoadMIB->getOperand(1).setIsKill(false);
auto *StoreMMO = *MI.memoperands_begin();
@@ -127,6 +143,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.addReg(DstAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, StoreMIB);
StoreMIB.addMemOperand(StoreMMO);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ebc4d6f2e379b..d5d1e1f44fb99 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5235,8 +5235,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// The lowering of a VGPR "as memory" indexed access relies on it having
// exactly one memory operand, since a sub-dword store synthesizes a load
// operand from it, and on the index carrying no subregister, since
- // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
- // is declared by the instruction itself, so it needs no check here.
+ // SITargetLowering::finalizeLowering rewrites it in place to M0.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 70f872c6d0662..1b66711c4fbe3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1180,7 +1180,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
let mayLoad = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Used for byte or short stores.
@@ -1193,7 +1192,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
let mayStore = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index 03646b899cbae..c5270bd219141 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -7,9 +7,10 @@
# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
#
-# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
-# virtual here and the implicit-def $m0 that instruction selection attached is
-# carried over to the whole-dword accesses the expansion creates.
+# The index and any M0 clobber are given to the pseudo beforehand, by
+# SITargetLowering::finalizeLowering, so the whole-dword accesses the expansion
+# creates carry both over: the index as their own index operand, and the clobber
+# as an implicit def where the subtarget indexes with the VGPR indexing mode.
---
name: lower_load_idx_bits_zext
@@ -21,7 +22,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +40,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +58,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +79,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +101,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +123,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
>From 7c517dac4d1c67f44392a03ef48b711eb350a848 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 14 Aug 2026 15:05:15 +0200
Subject: [PATCH 19/19] Reject under-aligned whole-dword VGPR-memory accesses
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 20 ++++++---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 21 ++++++----
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 42 +++++++++++++++----
3 files changed, 62 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0f88afbf20e37..92143056750de 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -559,14 +559,19 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
// memory access producing/consuming a ValSize-bit value at the given alignment.
// Whole-dword accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo)
-// are supported, as are 8-/16-bit accesses, including extending loads into a
-// 16- or 32-bit value.
+// are supported when dword aligned, as are 8-/16-bit accesses, including
+// extending loads into a 16- or 32-bit value.
//
// A sub-dword access is implemented as a bit-field extract from (or insert
// into) the dword containing it, so it must not straddle a dword boundary. An
// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
// natural alignment covers both, and is what lets the bit offset within the
// dword be computed from a possibly dynamic pointer.
+//
+// A whole-dword access addresses registers by the dword index pointer >> 2,
+// which discards the low two bits rather than accounting for them, so an
+// under-aligned one would silently access the containing dword instead of the
+// bytes asked for.
static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
Align Alignment) {
if (MemSize == 8 || MemSize == 16) {
@@ -580,6 +585,8 @@ static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
}
if (MemSize != ValSize)
return false;
+ if (Alignment < Align(4))
+ return false;
return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
@@ -3540,15 +3547,16 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
- // Reject anything else with a diagnostic instead of failing to legalize.
+ // Dword-aligned whole-dword and naturally aligned 8-/16-bit accesses are
+ // implemented. Reject anything else with a diagnostic instead of failing to
+ // legalize.
if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and naturally aligned 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9f334be564f7f..6a95656d31a42 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13206,18 +13206,18 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
- // Reject anything else with a diagnostic (replacing the value with poison)
- // instead of failing instruction selection. Both callers - operation
- // legalization and the pre-ISel combine - replace the node with this result,
- // so the diagnostic is emitted exactly once.
+ // Dword-aligned whole-dword and naturally aligned 8-/16-bit accesses are
+ // implemented. Reject anything else with a diagnostic (replacing the value
+ // with poison) instead of failing instruction selection. Both callers -
+ // operation legalization and the pre-ISel combine - replace the node with
+ // this result, so the diagnostic is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and naturally aligned 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13337,6 +13337,13 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
return DAG.getMergeValues({Value, LoadChain}, DL);
}
+ // The dword index below is the pointer shifted right by two, which discards
+ // the low two bits rather than accounting for them, so an under-aligned
+ // access would silently read or write the containing dword instead of the
+ // bytes asked for.
+ if (MemOp->getAlign() < Align(4))
+ return reportUnsupported();
+
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 6a8c7772716a1..5ebb2258784bb 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -4,26 +4,52 @@
; Accesses of the VGPR "as memory" address space (13) that are not implemented
; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
; rather than failing with "cannot select" / "unable to legalize" - or, worse,
-; silently generating wrong code. Whole-dword and naturally aligned 8-/16-bit
-; accesses are implemented; see as-vgpr-basic.ll and as-vgpr-bits.ll.
+; silently generating wrong code. Dword-aligned whole-dword and naturally
+; aligned 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
+; as-vgpr-bits.ll.
+
+; A whole-dword access addresses registers by the dword index pointer >> 2,
+; which discards the low two bits rather than accounting for them. An
+; under-aligned one would therefore access the dword containing the address
+; instead of the bytes asked for, reading or writing the wrong data with
+; nothing to show for it, so it has to be rejected here.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
+ %x = load i32, ptr addrspace(13) %p, align 1
+ ret i32 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
+ store i32 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
+; Alignment is required of the pointer, not of the accessed type: a 64-bit
+; access needs only the dword alignment the index computation relies on.
+; CHECK-NOT: in function load_i64_align4
+define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
+ %x = load i64, ptr addrspace(13) %p, align 4
+ ret i64 %x
+}
; An under-aligned 16-bit access may straddle a dword boundary, which the
; bit-field extract / insert lowering cannot express.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i32 @load_i16_align1(ptr addrspace(13) inreg %p) {
%x = load i16, ptr addrspace(13) %p, align 1
%y = zext i16 %x to i32
ret i32 %y
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i16_align1(ptr addrspace(13) inreg %p, i16 %v) {
store i16 %v, ptr addrspace(13) %p, align 1
ret void
}
; A sub-dword load extended into a value wider than a dword.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
%y = zext i8 %x to i64
@@ -31,20 +57,20 @@ define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
}
; A memory size that is neither a whole dword nor 8/16 bits.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i1 @load_i1(ptr addrspace(13) inreg %p) {
%x = load i1, ptr addrspace(13) %p
ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
store i1 %v, ptr addrspace(13) %p
ret void
}
; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
%x = load <14 x i32>, ptr addrspace(13) %p
ret <14 x i32> %x
More information about the llvm-branch-commits
mailing list