[llvm-branch-commits] [llvm] [AMDGPU] Add support for sub-dword types for address space 13 (PR #213223)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Sep 14 20:53:10 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/213223
>From e52f7f04ca01423a9d02163b8721413a0b36538b Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 12:39:47 +0300
Subject: [PATCH 01/22] Add support for sub-dword types for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 10 +
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 2 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 146 +++++-
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 201 ++++++++
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 9 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 6 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 18 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 11 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 134 +++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 18 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 82 ++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 449 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 69 +--
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
19 files changed, 1127 insertions(+), 56 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 199231e7eea89..569db8c4e9542 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -289,6 +289,9 @@ extern char &AMDGPURegBankLegalizeLegacyID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
+void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
+extern char &AMDGPULowerIdxOpsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -503,6 +506,13 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
+class AMDGPULowerIdxOpsPass
+ : public RequiredPassInfoMixin<AMDGPULowerIdxOpsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index f55bc70ca31f4..d2d58c5860dc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -292,6 +292,8 @@ def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD_BITS, SIreg_load_bits>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE_BITS, SIreg_store_bits>;
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 371c1c913fdc8..7f7c205780ad3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -560,10 +560,19 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") lowering handles a MemSize-bit access
-// producing a ValSize-bit value. Whole-dword only for now.
+// producing a ValSize-bit value. Whole-dword and 8-/16-bit, the latter
+// including extending loads into a 16- or 32-bit value.
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
- return MemSize == ValSize &&
- AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+ if (MemSize == 8 || MemSize == 16) {
+ if (ValSize == MemSize)
+ return true;
+ if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
+ return true;
+ return false;
+ }
+ if (MemSize != ValSize)
+ return false;
+ return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
/// Return true if a load or store of the type should be lowered with a bitcast
@@ -1810,6 +1819,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
{S16, ConstantPtr, S8, GlobalAlign8}})
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
+ })
+ // VGPR ("as memory") extending loads are custom-lowered to
+ // G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
+ // unsupported access is diagnosed cleanly rather than failing to
+ // legalize.
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
if (ST.hasFlatAddressSpace()) {
@@ -3479,34 +3495,38 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
-/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into
-/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE. Parallels LowerLoadStoreVGPR.
+/// Lower a G_LOAD / G_STORE / G_SEXTLOAD / G_ZEXTLOAD on AMDGPUAS::VGPR into
+/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE, or their _BITS forms for sub-dword
+/// accesses. Parallels LowerLoadStoreVGPR.
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
MachineMemOperand &MMO = **MI.memoperands_begin();
- const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ const unsigned Opcode = MI.getOpcode();
+ const bool IsStore = Opcode == AMDGPU::G_STORE;
Register ValReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
+ const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
const LLT ValTy = MRI.getType(ValReg);
const unsigned ValSize = ValTy.getSizeInBits();
// The selection patterns match the extended integer LLT, so build the index
// and the normalized value with integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Alignment is checked here rather than in the size predicate: the index is
- // the pointer >> 2, so an under-aligned access would silently reach the
- // containing dword. That is a property of the address, not of the size.
- if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
- ValSize) ||
- MMO.getAlign() < Align(4)) {
+ // Alignment is checked here rather than in the size predicate: it is a
+ // property of the address, not the size. A whole-dword access indexes by
+ // pointer >> 2, so an under-aligned one would silently reach the containing
+ // dword; the sub-dword path below computes a bit offset instead.
+ if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize) ||
+ (MemSize >= 32 && MMO.getAlign() < Align(4))) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword and 8-/16-bit loads and stores are "
+ "implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
@@ -3514,6 +3534,102 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (MemSize < 32) {
+ assert(MemSize == 8 || MemSize == 16);
+ assert(MemSize <= ValSize && ValSize <= 32);
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // known constant.
+ Register BaseReg = PtrReg;
+ int64_t Offset = 0;
+ if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
+ if (auto MaybeOff =
+ getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
+ BaseReg = PtrAdd->getBaseReg();
+ Offset = MaybeOff->Value.getSExtValue();
+ }
+ }
+
+ bool HaveConstantBitOffset = false;
+ int64_t ConstantBitOffsetVal = 0;
+ if (Offset == 0 && MMO.getAlign() >= Align(4)) {
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = 0;
+ } else {
+ auto &VT = *Helper.getValueTracking();
+ KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = (Offset & 3) * 8;
+ }
+ }
+
+ // Setup common registers.
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ Register BitOffsetReg;
+ if (HaveConstantBitOffset) {
+ BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
+ // offset, so masking the high bits is unnecessary.
+ auto Three = B.buildConstant(I32, 3);
+ BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
+ }
+
+ if (IsStore) {
+ LLT IValTy = LLT::integer(ValSize);
+ Register Value = ValReg;
+ if (IValTy != ValTy)
+ Value = B.buildBitcast(IValTy, Value).getReg(0);
+ if (ValSize != 32)
+ Value = B.buildAnyExt(I32, Value).getReg(0);
+
+ Register ShiftAmt = BitOffsetReg;
+ if (!HaveConstantBitOffset) {
+ auto Mask = B.buildConstant(I32, 31);
+ ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
+ }
+ Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
+
+ const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
+ const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
+ .addMemOperand(&MMO);
+ } else {
+ const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
+ const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ Register Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
+ {Index, BitWidthReg, BitOffsetReg, IsSExtReg})
+ .addMemOperand(&MMO)
+ .getReg(0);
+
+ if (ValTy != I32) {
+ LLT ResultTy = I32;
+ if (ValSize != 32) {
+ ResultTy = LLT::integer(ValSize);
+ Result = B.buildTrunc(ResultTy, Result).getReg(0);
+ }
+ if (ResultTy != ValTy)
+ Result = B.buildBitcast(ValTy, Result).getReg(0);
+ }
+
+ B.buildCopy(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+ }
+
const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
MachineInstrBuilder Two = B.buildConstant(I32, 2);
const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
@@ -3555,7 +3671,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
- if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ // G_LOAD as well as the extending loads (G_SEXTLOAD / G_ZEXTLOAD) are
+ // custom-lowered; the latter only exist for sub-dword accesses.
+ if (AddrSpace == AMDGPUAS::VGPR)
return lowerLoadStoreVGPR(Helper, MI);
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
new file mode 100644
index 0000000000000..95cfe42df7e6d
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -0,0 +1,201 @@
+//===- AMDGPULowerIdxOps.cpp - Expand sub-dword VGPR-memory accesses -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Expand the sub-dword VGPR "as memory" (address space 13) pseudos into a
+/// whole-dword indexed access plus a bit-field extract or insert:
+///
+/// V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+///
+/// A sub-dword store is therefore a read-modify-write of the containing dword.
+/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
+/// here take part in the usual M0 setup, and before register allocation
+/// because it introduces new virtual registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-lower-idx-ops"
+
+namespace {
+
+class LowerIdxOps {
+public:
+ LowerIdxOps(MachineFunction &MF)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ MRI(&MF.getRegInfo()) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ void lowerLoadIdxBits(MachineInstr &MI);
+ void lowerStoreIdxBits(MachineInstr &MI);
+
+ const SIInstrInfo *TII;
+ MachineRegisterInfo *MRI;
+};
+
+void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
+
+ const bool IsSigned = MI.getOperand(5).getImm() != 0;
+ const MCInstrDesc &II =
+ TII->get(IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPUMI::VLoadIdxInst::getOpcodeForBitWidth(32)),
+ SrcAReg)
+ .add(LoadIdx.getIdxOp())
+ .add(LoadIdx.getOffsetOp());
+ LoadMIB.addMemOperand(*MI.memoperands_begin());
+ // Match what instruction selection does for a whole-dword access with a
+ // register index: record that the M0 write implied by the eventual movrel
+ // clobbers M0 (see AMDGPUAssignIdxToM0).
+ if (LoadIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Extract the accessed bits out of it.
+ MachineOperand BitOffset = MI.getOperand(4);
+ if (BitOffset.isImm())
+ BitOffset.setImm(BitOffset.getImm() & 31);
+
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ .addReg(SrcAReg)
+ .add(BitOffset)
+ .add(MI.getOperand(3)); // bitsize
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineFunction *MF = MBB->getParent();
+ auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB = BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ // The index is read again by the store below, so it does not die here.
+ LoadMIB->getOperand(1).setIsKill(false);
+ auto *StoreMMO = *MI.memoperands_begin();
+ // Synthesize a load MMO from the store's.
+ auto NewFlags = MachineMemOperand::MOLoad;
+ NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
+ LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ if (StoreIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Insert the stored bits into it.
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
+ .add(StoreIdx.getOperand(3)) // mask
+ .addReg(StoreIdx.getDataOp().getReg())
+ .addReg(SrcAReg);
+
+ // Write the dword back.
+ auto StoreMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_STORE_IDX_B32))
+ .addReg(DstAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ StoreMIB.addMemOperand(StoreMMO);
+ if (StoreIdx.getIdxOp().isReg())
+ StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+bool LowerIdxOps::run(MachineFunction &MF) {
+ bool Changed = false;
+
+ LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (auto MII = MBB.begin(), E = MBB.end(); MII != E;) {
+ MachineInstr &MI = *MII++;
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_BITS:
+ lowerLoadIdxBits(MI);
+ Changed = true;
+ break;
+ case AMDGPU::V_STORE_IDX_BITS:
+ lowerStoreIdxBits(MI);
+ Changed = true;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ return Changed;
+}
+
+class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return LowerIdxOps(MF).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Lower Idx Ops"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!LowerIdxOps(MF).run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPULowerIdxOpsLegacy::ID = 0;
+
+char &llvm::AMDGPULowerIdxOpsID = AMDGPULowerIdxOpsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPULowerIdxOpsLegacy, DEBUG_TYPE, "AMDGPU Lower Idx Ops",
+ false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
index 697c63e2079d1..970e9916f14dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -19,14 +19,19 @@ using namespace llvm;
using namespace AMDGPUMI;
unsigned VLoadStoreIdxInst::getBitWidth() const {
+ unsigned Opc = getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ llvm_unreachable("V_LOAD/STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
if (!Info)
llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
return Info->BitWidth;
}
int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_LOAD_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
if (!Info)
@@ -41,6 +46,8 @@ unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
}
int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
if (!Info)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index 24e24aab53152..e99594062871a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -37,7 +37,10 @@ class VLoadStoreIdxInst : public MachineInstr {
unsigned getBitWidth() const;
static bool classof(const MachineInstr *MI) {
- return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc) != nullptr;
}
};
@@ -47,8 +50,11 @@ class VLoadIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && !Info->IsStore;
}
};
@@ -59,8 +65,11 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && Info->IsStore;
}
};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 29e9046e4f8ca..488bfdb2ee54c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -120,6 +120,7 @@ MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-lower-idx-ops", AMDGPULowerIdxOpsPass())
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
MACHINE_FUNCTION_PASS("amdgpu-postlegalizer-combiner",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7c81aafb141ed..b5c5b72e8c1fa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1399,6 +1399,12 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForGOpcs({G_AMDGPU_REG_STORE})
.Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD_BITS})
+ .Any({{BRC}, {{VgprBRC}, {Sgpr32_WF, Sgpr32, Vgpr32, Sgpr32}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE_BITS})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF, Vgpr32}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index f2bd7eee43c74..84c8eb2b59c95 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3089,7 +3089,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// The dword index (operand 1) must be uniform; a divergent index needs a
// waterfall loop.
applyDefaultMapping(OpdMapper);
@@ -4497,9 +4499,21 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
break;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ const unsigned Opc = MI.getOpcode();
+ if (Opc == AMDGPU::G_AMDGPU_REG_LOAD_BITS) {
+ // {dst, index, bitsize, bitoffset, is_sext_load}
+ OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+ OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+ } else if (Opc == AMDGPU::G_AMDGPU_REG_STORE_BITS) {
+ // {value, index, mask}
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ }
break;
}
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b8a1e4a656fd8..bc2d80bbf675d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -695,6 +695,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizeLegacyPass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
+ initializeAMDGPULowerIdxOpsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1837,6 +1838,12 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert. The index of the access it expands has
+ // already been copied into M0 by SITargetLowering::finalizeLowering, and the
+ // whole-dword access created here inherits it.
+ addPass(&AMDGPULowerIdxOpsID);
+
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
@@ -2647,6 +2654,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert; see the comment in GCNPassConfig.
+ addMachineFunctionPass(AMDGPULowerIdxOpsPass(), PMW);
+
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b18ca6bd24040..5c9f8a2063847 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -90,6 +90,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUMemoryUtils.cpp
AMDGPUCoExecSchedStrategy.cpp
AMDGPUIGroupLP.cpp
+ AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c887c8340de08..b0c8dd6926439 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13602,19 +13602,125 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword and 8-/16-bit loads and stores are "
+ "implemented",
DL.getDebugLoc()));
SmallVector<EVT, 2> ResultTypes(Op->values());
return DAG.getErrorMergeValues(ResultTypes, MemOp->getChain(), DL);
};
- if (BitWidth < 32)
- return reportUnsupported();
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (BitWidth < 32) {
+ if (BitWidth != 8 && BitWidth != 16)
+ return reportUnsupported();
+
+ // Bail out for sub-dword types we cannot handle.
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
+ Load->getValueType(0).getSizeInBits() > 32)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore() &&
+ Store->getValue().getValueType() != MVT::i16 &&
+ Store->getValue().getValueType() != MVT::i32)
+ return reportUnsupported();
+ }
+
+ SDValue Ptr = MemOp->getBasePtr();
+
+ // Split the address into base and constant offset.
+ SDValue Base;
+ int32_t Offset;
+ if (DAG.isBaseWithConstantOffset(Ptr)) {
+ Base = Ptr.getOperand(0);
+ Offset = cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue();
+ } else if (auto *C = dyn_cast<ConstantSDNode>(Ptr)) {
+ Base = DAG.getConstant(0, DL, MVT::i32);
+ Offset = C->getSExtValue();
+ } else {
+ Base = Ptr;
+ Offset = 0;
+ }
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // constant.
+ KnownBits BaseKB;
+ if (Offset == 0 && MemOp->getAlign() >= Align(4)) {
+ BaseKB = KnownBits::makeConstant(APInt::getZero(2));
+ } else {
+ BaseKB = DAG.computeKnownBits(Base).trunc(2);
+ }
- // The index is the pointer >> 2, so an under-aligned access would silently
- // reach the containing dword rather than the bytes asked for.
+ SDValue BitOffset;
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ BitOffset = DAG.getConstant((Offset & 3) * 8, DL, MVT::i32);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the least 5 bits of the bit
+ // offset, so we do not have to mask off the high bits.
+ BitOffset = DAG.getNode(ISD::SHL, DL, MVT::i32, Ptr,
+ DAG.getConstant(3, DL, MVT::i32));
+ }
+
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
+ DAG.getConstant(2, DL, MVT::i32));
+ SDValue BitSizeImm = DAG.getConstant(BitWidth, DL, MVT::i32);
+ SDValue Chain = MemOp->getChain();
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ EVT ValVT = Value.getValueType();
+ if (!ValVT.isScalarInteger())
+ Value = DAG.getNode(
+ ISD::BITCAST, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ValVT.getSizeInBits()), Value);
+ Value = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Value);
+
+ // ISD::SHL produces poison for shift amounts >= bit width, which generic
+ // DAG combines rely on. We need the mask for correctness, but it should
+ // get optimized away during final instruction selection.
+ SDValue ShiftOffset = DAG.getNode(ISD::AND, DL, MVT::i32, BitOffset,
+ DAG.getConstant(31, DL, MVT::i32));
+ Value = DAG.getNode(ISD::SHL, DL, MVT::i32, Value, ShiftOffset);
+
+ SDValue MaskBase = DAG.getConstant((1u << BitWidth) - 1, DL, MVT::i32);
+ SDValue Mask = DAG.getNode(ISD::SHL, DL, MVT::i32, MaskBase, ShiftOffset);
+
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE_BITS, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index, Mask}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ bool IsSExt = LoadOp->getExtensionType() == ISD::SEXTLOAD;
+ SDValue IsSExtImm = DAG.getConstant(IsSExt ? 1 : 0, DL, MVT::i32);
+ SDValue Value = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD_BITS, DL, DAG.getVTList(MVT::i32, MVT::Other),
+ {Chain, Index, BitSizeImm, BitOffset, IsSExtImm}, MemVT,
+ LoadOp->getMemOperand());
+ EVT ResVT = LoadOp->getValueType(0);
+ if (ResVT == MVT::i32)
+ return Value;
+
+ SDValue LoadChain = Value.getValue(1);
+ unsigned ResWidth = ResVT.getSizeInBits();
+ if (ResWidth < 32)
+ Value =
+ DAG.getNode(ISD::TRUNCATE, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ResWidth), Value);
+ if (Value.getValueType() != ResVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
+ return DAG.getMergeValues({Value, LoadChain}, DL);
+ }
+ // Whole-dword accesses index by pointer >> 2, so an under-aligned one would
+ // silently reach the containing dword. The sub-dword path above computes a
+ // bit offset and carries its own alignment rule.
if (MemOp->getAlign() < Align(4))
return reportUnsupported();
+
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
@@ -19478,7 +19584,9 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
case AMDGPUISD::REG_LOAD:
- case AMDGPUISD::REG_STORE: {
+ case AMDGPUISD::REG_STORE:
+ case AMDGPUISD::REG_LOAD_BITS:
+ case AMDGPUISD::REG_STORE_BITS: {
const SIMachineFunctionInfo *MFI =
DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
unsigned NumAddressableVGPRs =
@@ -19486,7 +19594,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0;
+ unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
@@ -19494,10 +19602,22 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case AMDGPUISD::REG_STORE:
IndexOpIdx = 2;
break;
+ case AMDGPUISD::REG_LOAD_BITS:
+ IndexOpIdx = 1;
+ BitOffOpIdx = 3;
+ break;
+ case AMDGPUISD::REG_STORE_BITS:
+ IndexOpIdx = 2;
+ break;
}
if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
return SDValue(N, 0);
+ if (BitOffOpIdx) {
+ APInt BitOffMask = APInt::getLowBitsSet(32, 5);
+ if (SimplifyDemandedBits(N->getOperand(BitOffOpIdx), BitOffMask, DCI))
+ return SDValue(N, 0);
+ }
break;
}
case ISD::LOAD: {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 294ac6d77e012..9e3dbb708e63f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11251,7 +11251,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
// Always divergent: it reads the wave's per-lane registers, so even a uniform
// index yields a per-lane value.
- if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD ||
+ Opcode == AMDGPU::G_AMDGPU_REG_LOAD_BITS)
return ValueUniformity::NeverUniform;
// Loads from the private and flat address spaces are divergent, because
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 075c4df4891f1..c59deada89baf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -69,6 +69,24 @@ def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+// VGPR address space sub-dword (8/16-bit) load/store with dword index. The
+// access is described by the dword index plus the bit position within that
+// dword; the store form carries a pre-shifted value and an insert mask.
+def SDTRegIdxBitsLoad : SDTypeProfile<1, 4,
+ [SDTCisInt<0>, // result (i32 or i16)
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>, // bitsize
+ SDTCisVT<3, i32>, // bitoffset
+ SDTCisVT<4, i32>]>; // is_sext_load
+def SDTRegIdxBitsStore : SDTypeProfile<0, 3,
+ [SDTCisInt<0>, // data (i32 or i16), pre-shifted insert value
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>]>; // insert mask
+def SIreg_load_bits : SDNode<"AMDGPUISD::REG_LOAD_BITS", SDTRegIdxBitsLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store_bits : SDNode<"AMDGPUISD::REG_STORE_BITS", SDTRegIdxBitsStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index da9d8752b1e53..0d65c481e4441 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1146,6 +1146,70 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+// Used for byte or short loads.
+// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
+// sign-extended loads.
+//
+// Only the least 5 bits of $bitoffset are significant, the rest are ignored.
+// Only offset values of 0, 8, 16, or 24 are meaningful (after masking to the
+// 5 LSBs).
+//
+// Only bitsize of 8 or 16 bits are supported and must be a compile-time
+// constant.
+class V_LOAD_IDX_BITS<RegisterClass data_rc>
+ : VPseudoInstSI<(outs data_rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
+ VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
+
+// Used for byte or short stores.
+//
+// $data holds the pre-shifted insert value and $mask holds the insert mask
+// that is used when lowering to V_BFI_B32.
+class V_STORE_IDX_BITS<RegisterClass data_op_rc>
+ : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
+
+// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
+// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
+// shape folds a constant dword offset into the pseudo's $offset operand.
+multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
+ Instruction store_inst, ValueType vt> {
+ def : GCNPat<
+ (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+ def : GCNPat<
+ (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (store_inst $data, $idx, imm:$offset, $mask)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
+ (store_inst $data, $idx, 0, $mask)>;
+}
+
+defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4912,6 +4976,24 @@ def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
let mayStore = 1;
}
+// Sub-dword (8/16-bit) counterparts of the above. In addition to the dword
+// index they carry the bit position of the access within that dword, and are
+// lowered to a read-modify-write of the whole dword (see AMDGPULowerIdxOps).
+def G_AMDGPU_REG_LOAD_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index, type1:$bitsize,
+ type1:$bitoffset, type1:$is_sext_load);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index, type1:$mask);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
new file mode 100644
index 0000000000000..31c9b44e7236d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -0,0 +1,449 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
+; address space (13). A load reads the containing dword with an M0-relative
+; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
+; read-modify-write of that dword using v_bfi_b32 (see AMDGPULowerIdxOps).
+
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+define i32 @load_i8_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i8_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = sext i8 %x to i32
+ ret i32 %y
+}
+
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+define i32 @load_i16_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i16_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = sext i16 %x to i32
+ ret i32 %y
+}
+
+; A dword-aligned access has a statically known bit offset of zero.
+define i32 @load_i8_aligned(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+; A constant byte offset folds into a constant bit offset.
+define i32 @load_i8_const_offset(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i8_const_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 6
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_const_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 6
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 6
+ %x = load i8, ptr addrspace(13) %q, align 2
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+; GFX12-SDAG-LABEL: store_i16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-LABEL: store_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; A divergent index is made uniform by a waterfall loop.
+define i32 @load_i8_divergent(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 31, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-GISEL-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 0cbda8e8ff646..28dcb1b27f86e 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,49 +1,54 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are not implemented
-; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
-; rather than failing with "cannot select" / "unable to legalize" - or, worse,
-; silently generating wrong code.
-
-; Sub-dword (8/16-bit) accesses are not yet implemented; support lands in a
-; later change.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; Accesses of the VGPR "as memory" address space (13) that are still not
+; implemented must be rejected with a clean diagnostic on both SelectionDAG and
+; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
+; Dword-aligned whole-dword and 8-/16-bit accesses are implemented; see
+; as-vgpr-basic.ll and
+; as-vgpr-bits.ll.
+
+; A sub-dword load extended into a value wider than a dword.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
- ret i8 %x
+ %y = zext i8 %x to i64
+ ret i64 %y
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define i16 @load_i16(ptr addrspace(13) inreg %p) {
- %x = load i16, ptr addrspace(13) %p
- ret i16 %x
+; A memory size that is neither a whole dword nor 8/16 bits.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define i1 @load_i1(ptr addrspace(13) inreg %p) {
+ %x = load i1, ptr addrspace(13) %p
+ ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
- store i8 %v, ptr addrspace(13) %p
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
+ store i1 %v, ptr addrspace(13) %p
ret void
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
- store i16 %v, ptr addrspace(13) %p
- ret void
+; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
+ %x = load <14 x i32>, ptr addrspace(13) %p
+ ret <14 x i32> %x
}
-; An access addresses registers by the dword index pointer >> 2, which discards
-; the low two bits rather than accounting for them. An under-aligned one would
-; therefore reach the dword containing the address instead of the bytes asked
-; for - the same code as a correctly aligned access, reading the wrong data with
-; nothing to show for it.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+; A whole-dword access addresses registers by the dword index pointer >> 2,
+; which discards the low two bits rather than accounting for them. An
+; under-aligned one would therefore reach the dword containing the address
+; instead of the bytes asked for - the same code as a correctly aligned access,
+; reading the wrong data with nothing to show for it. A sub-dword access
+; computes a bit offset instead, and is held only to its own natural alignment.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
%x = load i32, ptr addrspace(13) %p, align 1
ret i32 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
store i32 %v, ptr addrspace(13) %p, align 1
ret void
@@ -56,3 +61,11 @@ define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
%x = load i64, ptr addrspace(13) %p, align 4
ret i64 %x
}
+
+; And a byte access at align 1 is fine, which is what the whole-dword-only
+; restriction must not catch.
+; CHECK-NOT: in function load_i8_align1
+define i8 @load_i8_align1(ptr addrspace(13) inreg %p) {
+ %x = load i8, ptr addrspace(13) %p, align 1
+ ret i8 %x
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 89f72e559ba52..ba683ded7e575 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,6 +68,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -215,6 +216,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-lower-idx-ops
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -402,6 +404,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-lower-idx-ops
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index bb7ed3b58f8af..19d8cbae5c1e8 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -354,6 +355,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -683,6 +685,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -1016,6 +1019,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Lower Idx Ops
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1365,6 +1369,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Lower Idx Ops
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From d81d1246ddfec81881c38811bc65124a6ff61119 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:14:32 +0300
Subject: [PATCH 02/22] Correctly diagnose the mis-aligned sub-dowrd accesses
instead of miscompiling them
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 28 +++---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 10 ++-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 43 +++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 88 +++++++++++--------
4 files changed, 119 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 7f7c205780ad3..db247727bac50 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -560,10 +560,18 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") lowering handles a MemSize-bit access
-// producing a ValSize-bit value. Whole-dword and 8-/16-bit, the latter
-// including extending loads into a 16- or 32-bit value.
-static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+// producing a ValSize-bit value at this alignment: whole-dword when dword
+// aligned, and 8-/16-bit when naturally aligned, including extending loads.
+//
+// A sub-dword access is a bit-field extract from the dword containing it, so it
+// must not straddle a dword boundary; natural alignment guarantees that. A
+// whole-dword access indexes by pointer >> 2, so an under-aligned one would
+// silently reach the containing dword.
+static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment) {
if (MemSize == 8 || MemSize == 16) {
+ if (Alignment < Align(MemSize / 8))
+ return false;
if (ValSize == MemSize)
return true;
if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
@@ -572,6 +580,8 @@ static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
}
if (MemSize != ValSize)
return false;
+ if (Alignment < Align(4))
+ return false;
return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
@@ -3515,18 +3525,14 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// and the normalized value with integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Alignment is checked here rather than in the size predicate: it is a
- // property of the address, not the size. A whole-dword access indexes by
- // pointer >> 2, so an under-aligned one would silently reach the containing
- // dword; the sub-dword path below computes a bit offset instead.
- if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize) ||
- (MemSize >= 32 && MMO.getAlign() < Align(4))) {
+ // Diagnose an unsupported access rather than failing to legalize.
+ if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword and 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b0c8dd6926439..b604a415b5713 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13602,8 +13602,8 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword and 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
DL.getDebugLoc()));
SmallVector<EVT, 2> ResultTypes(Op->values());
return DAG.getErrorMergeValues(ResultTypes, MemOp->getChain(), DL);
@@ -13616,6 +13616,12 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
if (BitWidth != 8 && BitWidth != 16)
return reportUnsupported();
+ // The access becomes a bit-field extract from (or insert into) the dword
+ // containing it, so it must not straddle a dword boundary. An 8-bit access
+ // never can; a 16-bit one only if it is 2-byte aligned.
+ if (MemOp->getAlign() < Align(BitWidth / 8))
+ return reportUnsupported();
+
// Bail out for sub-dword types we cannot handle.
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 31c9b44e7236d..4b1703c1b6043 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -447,3 +447,46 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
store i8 %v, ptr addrspace(13) %p
ret void
}
+
+; A 2-byte aligned 16-bit access sits at bit offset 0 or 16, so it always fits
+; within the containing dword even though the offset is not known statically.
+; (An under-aligned 16-bit access is rejected; see as-vgpr-unsupported.ll.)
+define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i16_offset2:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i16_offset2:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 2
+ %x = load i16, ptr addrspace(13) %q, align 2
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 28dcb1b27f86e..22dc236753367 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,15 +1,55 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are still not
-; implemented must be rejected with a clean diagnostic on both SelectionDAG and
-; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
-; Dword-aligned whole-dword and 8-/16-bit accesses are implemented; see
-; as-vgpr-basic.ll and
+; Accesses of the VGPR "as memory" address space (13) that are not implemented
+; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
+; rather than failing with "cannot select" / "unable to legalize" - or, worse,
+; silently generating wrong code. Dword-aligned whole-dword and naturally
+; aligned 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
; as-vgpr-bits.ll.
+; A whole-dword access addresses registers by the dword index pointer >> 2,
+; which discards the low two bits rather than accounting for them. An
+; under-aligned one would therefore access the dword containing the address
+; instead of the bytes asked for, reading or writing the wrong data with
+; nothing to show for it, so it has to be rejected here.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
+ %x = load i32, ptr addrspace(13) %p, align 1
+ ret i32 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
+ store i32 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
+; Alignment is required of the pointer, not of the accessed type: a 64-bit
+; access needs only the dword alignment the index computation relies on.
+; CHECK-NOT: in function load_i64_align4
+define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
+ %x = load i64, ptr addrspace(13) %p, align 4
+ ret i64 %x
+}
+
+; An under-aligned 16-bit access may straddle a dword boundary, which the
+; bit-field extract / insert lowering cannot express.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i16_align1(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p, align 1
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i16_align1(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
; A sub-dword load extended into a value wider than a dword.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
%y = zext i8 %x to i64
@@ -17,53 +57,27 @@ define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
}
; A memory size that is neither a whole dword nor 8/16 bits.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i1 @load_i1(ptr addrspace(13) inreg %p) {
%x = load i1, ptr addrspace(13) %p
ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
store i1 %v, ptr addrspace(13) %p
ret void
}
; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
%x = load <14 x i32>, ptr addrspace(13) %p
ret <14 x i32> %x
}
-; A whole-dword access addresses registers by the dword index pointer >> 2,
-; which discards the low two bits rather than accounting for them. An
-; under-aligned one would therefore reach the dword containing the address
-; instead of the bytes asked for - the same code as a correctly aligned access,
-; reading the wrong data with nothing to show for it. A sub-dword access
-; computes a bit offset instead, and is held only to its own natural alignment.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
-define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
- %x = load i32, ptr addrspace(13) %p, align 1
- ret i32 %x
-}
-
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
-define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
- store i32 %v, ptr addrspace(13) %p, align 1
- ret void
-}
-
-; Alignment is required of the pointer, not of the accessed type: a 64-bit
-; access needs only the dword alignment the index computation relies on.
-; CHECK-NOT: in function load_i64_align4
-define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
- %x = load i64, ptr addrspace(13) %p, align 4
- ret i64 %x
-}
-
-; And a byte access at align 1 is fine, which is what the whole-dword-only
-; restriction must not catch.
+; And an 8-bit access is held only to its own natural alignment, which is one
+; byte - the whole-dword rule must not catch it.
; CHECK-NOT: in function load_i8_align1
define i8 @load_i8_align1(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p, align 1
>From 46e362fb69e63b91dab30e875523b547e62768b2 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:12:39 +0300
Subject: [PATCH 03/22] Run AMDGPULowerIdxOps for optnone functions in
AMDGPULowerIdxOps
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 5 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 110 ++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 95cfe42df7e6d..e7d3a1e8bf172 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -168,8 +168,9 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: nothing else expands the
+ // sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
+ // therefore must not be skipped for optnone functions.
return LowerIdxOps(MF).run(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4b1703c1b6043..66790a4a155fe 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -490,3 +490,113 @@ define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
%y = zext i16 %x to i32
ret i32 %y
}
+
+; Expanding these pseudos is required lowering, so it has to happen for optnone
+; functions too - clang marks every function optnone at -O0. Nothing else can
+; lower them, so if the expansion is skipped AMDGPULowerVGPREncoding is left
+; with a pseudo it cannot handle.
+define i32 @load_i8_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-SDAG-LABEL: load_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_and_b32_e64 v0, v0, s0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnone {
+; GFX12-SDAG-LABEL: store_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ; kill: def $vgpr1_lo16 killed $vgpr0 killed $exec
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
>From d4cf214a11cb8dd3f809f042ca4808b2fd70f6a1 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:50:21 +0300
Subject: [PATCH 04/22] Add a MIR test for AMDGPULowerIdxOps
---
.../as-vgpr-lower-idx-ops.mir | 134 ++++++++++++++++++
1 file changed, 134 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
new file mode 100644
index 0000000000000..c991990db933e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+
+# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
+# AMDGPULowerIdxOps, in isolation:
+#
+# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+#
+# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
+# virtual here and the implicit-def $m0 that instruction selection attached is
+# carried over to the whole-dword accesses the expansion creates.
+
+---
+name: lower_load_idx_bits_zext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_zext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_sext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_sext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 1, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_16_offset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_16_offset
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 3, 16, 16, 0, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+# A dynamic bit offset is passed straight through to the bit-field extract;
+# only its low 5 bits are significant.
+---
+name: lower_load_idx_bits_dynamic_bitoffset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_dynamic_bitoffset
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, %1, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2
+...
+
+# The store is a read-modify-write of the containing dword. The index is read
+# again by the store, so it must not be killed by the load.
+---
+name: lower_store_idx_bits
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_store_idx_bits
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ V_STORE_IDX_BITS %1, killed %0, 0, 255, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ S_ENDPGM 0
+...
+
+---
+name: lower_store_idx_bits_dynamic_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: lower_store_idx_bits_dynamic_mask
+ ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = COPY $vgpr1
+ V_STORE_IDX_BITS %1, %0, 2, %2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ S_ENDPGM 0
+...
>From 9bbc0bf72b98d2cdfbbcfab8bfa1c15ba281baad Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:51:08 +0300
Subject: [PATCH 05/22] Verify operand invariants of the VGPR-memory indexed
accesses
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 28 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 26 +++++++++++++++++
2 files changed, 54 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9e3dbb708e63f..347879b5f7e4f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5475,6 +5475,34 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
+ // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
+ // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
+ // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
+ // access has to record that the move clobbers M0, otherwise it could be
+ // separated from the code that sets M0 up.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (MI.getNumMemOperands() != 1) {
+ ErrInfo = "v_load/store_idx should have exactly one memory operand.";
+ return false;
+ }
+
+ const MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg()) {
+ if (IdxOp.getSubReg() != 0) {
+ ErrInfo =
+ "v_load/store_idx register index must not have a subregister.";
+ return false;
+ }
+
+ if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
+ !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
+ ErrInfo = "v_load/store_idx with register index must have implicit-def "
+ "$m0 on movrel subtargets.";
+ return false;
+ }
+ }
+ }
+
// Make sure the register classes are correct.
for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
new file mode 100644
index 0000000000000..bec5ef54aa203
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -0,0 +1,26 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
+# access ends up in M0, which the v_movrel[sd] emitted by
+# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
+# index to M0, the access must record that the eventual move clobbers M0 -
+# otherwise it could be separated from the code that sets M0 up. Check that the
+# machine verifier rejects a register index without that implicit-def, for both
+# the whole-dword and the sub-dword pseudos.
+
+# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
+
+---
+name: missing_m0_implicit_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
+ %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From 5a3fe8ffb22c6c73bfca245f89b1e5a65a5af01a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:59:26 +0300
Subject: [PATCH 06/22] Test the bit-offset half of the VGPR-memory
demanded-bits combine
---
.../as-vgpr-index-demanded-bits.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index db1d8ad0bd4f6..352b35d6aac49 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -7,6 +7,11 @@
; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
; path the mask must not survive into the M0 index computation.
+;
+; The same combine also trims the bit offset of a sub-dword access, which needs
+; only its low 5 bits to name a bit position within the containing dword. That
+; half matters independently: a mask can be dead for the bit offset while the
+; index computation keeps it alive, and vice versa.
define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-SDAG-LABEL: load_masked_index:
@@ -47,5 +52,79 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
store i32 %val, ptr addrspace(13) %ptr
ret void
}
+
+; A dynamic pointer masked with 0x3ff: the mask is dead both for the index and
+; for the bit offset, so no s_and may remain.
+define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %m = and i32 %sum, 1023
+ %ptr = inttoptr i32 %m to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
+
+; Setting a high bit of the pointer cannot affect the 5-bit bit offset either,
+; so no bit-set of the shifted offset may remain.
+define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_or_high_bit:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_or_high_bit:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %o = or i32 %sum, 65536
+ %ptr = inttoptr i32 %o to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
>From ebf63efbd35db65f23c6529e0f7f1e4ac0de96d8 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:13:46 +0300
Subject: [PATCH 07/22] Simplify the sub-dword VGPR-memory pseudo definitions
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 57 +++++++++++-------------
1 file changed, 25 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 0d65c481e4441..89d7d74dac86b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1156,10 +1156,12 @@ defm : VRegIdxLoadStorePat<vt>;
//
// Only bitsize of 8 or 16 bits are supported and must be a compile-time
// constant.
-class V_LOAD_IDX_BITS<RegisterClass data_rc>
- : VPseudoInstSI<(outs data_rc:$data),
- (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
- VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+// The lowering normalizes the loaded value to a dword, so only a 32-bit $data
+// form is needed.
+def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
+ (ins SReg_32:$idx, i32imm:$offset,
+ i32imm:$bitsize, VSrc_b32:$bitoffset,
+ i32imm:$is_sext_load)> {
let mayLoad = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1167,15 +1169,13 @@ class V_LOAD_IDX_BITS<RegisterClass data_rc>
let Defs = [M0];
}
-def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
-
// Used for byte or short stores.
//
// $data holds the pre-shifted insert value and $mask holds the insert mask
// that is used when lowering to V_BFI_B32.
-class V_STORE_IDX_BITS<RegisterClass data_op_rc>
- : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
- i32imm:$offset, VSrc_b32:$mask)> {
+def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
+ (ins VGPR_32:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1183,32 +1183,25 @@ class V_STORE_IDX_BITS<RegisterClass data_op_rc>
let Defs = [M0];
}
-def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
-
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
// shape folds a constant dword offset into the pseudo's $offset operand.
-multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
- Instruction store_inst, ValueType vt> {
- def : GCNPat<
- (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
- (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
- imm:$is_sext_load)>;
- def : GCNPat<
- (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
- (store_inst $data, $idx, imm:$offset, $mask)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
- (store_inst $data, $idx, 0, $mask)>;
-}
-
-defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+def : GCNPat<
+ (i32 (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+def : GCNPat<
+ (i32 (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, imm:$offset, $mask)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, i32:$idx, i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, 0, $mask)>;
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
>From c7a5021bf93be344c7c42ac33ab7830fb7d60752 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:24:43 +0300
Subject: [PATCH 08/22] Cover the remaining VGPR-memory operand invariants
---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 38 +++++++++++++------
1 file changed, 26 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
index bec5ef54aa203..db32b6957abb3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -1,26 +1,40 @@
# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
-# access ends up in M0, which the v_movrel[sd] emitted by
-# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
-# index to M0, the access must record that the eventual move clobbers M0 -
-# otherwise it could be separated from the code that sets M0 up. Check that the
-# machine verifier rejects a register index without that implicit-def, for both
-# the whole-dword and the sub-dword pseudos.
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
+# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
+# rewritten the index to M0, the access must record that the eventual move
+# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
+# The lowering also relies on the access having exactly one memory operand (it
+# synthesizes a load operand from it for a sub-dword store), and on the index
+# carrying no subregister, since it is rewritten in place to M0.
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
---
-name: missing_m0_implicit_def
+name: operand_invariants
tracksRegLiveness: true
body: |
bb.0:
- liveins: $sgpr0, $vgpr0
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
- %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
+
+ ; Missing implicit-def $m0, for each of the four pseudo forms.
+ %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %2, implicit %3
+ S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
...
>From da8bc12a32ac0a00d90df19282af0f550b01d2d4 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:07:38 +0300
Subject: [PATCH 09/22] Also build the sub-dword VGPR-memory tests for wave64
---
llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 66790a4a155fe..992b737009947 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,6 +2,14 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
+; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
+; sub-dword code is the same there modulo register allocation, so only compile
+; and verify rather than checking the output a second time. What differs is the
+; waterfall around a divergent access, which uses a 64-bit exec mask.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
>From f5ceabbbf0a4de8a72934f1ed8a8713fc4127db7 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:28:47 +0300
Subject: [PATCH 10/22] Test sub-dword VGPR-memory accesses across control flow
---
.../as-vgpr-bits-control-flow.ll | 316 ++++++++++++++++++
1 file changed, 316 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
new file mode 100644
index 0000000000000..73edbbb7855e5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
+; across control flow. A sub-dword store is a read-modify-write of the dword
+; containing it, which makes it sensitive to how it is placed relative to other
+; accesses of the same dword, to loop bodies, and to the exec mask.
+
+; Two stores into different bytes of the same dword. Each is a full
+; read-modify-write, so the second has to observe the first: if the two dword
+; loads were merged, the first byte would be lost.
+define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
+; GFX12-SDAG-LABEL: two_bytes_same_dword:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: two_bytes_same_dword:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 1
+ store i8 %a, ptr addrspace(13) %p
+ store i8 %b, ptr addrspace(13) %q
+ ret void
+}
+
+; A store inside a loop. The pointer deliberately depends on the induction
+; variable: with a loop-invariant pointer the whole access is hoisted out and
+; nothing inside the loop is exercised. M0 has to be re-established for each
+; access on every iteration.
+define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
+; GFX12-SDAG-LABEL: store_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ store i8 %v, ptr addrspace(13) %q
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Likewise for a load, accumulating so the result cannot be folded away.
+define i32 @load_in_loop(ptr addrspace(13) inreg %p, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: .LBB2_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: .LBB2_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ %x = load i8, ptr addrspace(13) %q
+ %z = zext i8 %x to i32
+ %acc.next = add i32 %acc, %z
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; A store under divergent control flow: the read-modify-write must stay inside
+; the conditional region so that lanes which did not take the branch keep their
+; dword unmodified.
+define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
+; GFX12-SDAG-LABEL: store_divergent_if:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
+; GFX12-SDAG-NEXT: ; %bb.1: ; %then
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_divergent_if:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-GISEL-NEXT: ; %bb.1: ; %then
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %c = icmp ult i32 %tid, 8
+ br i1 %c, label %then, label %exit
+
+then:
+ store i8 %v, ptr addrspace(13) %p
+ br label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From 9ec9403b9cebb807d700050f0f85ca31af7c216a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:59:30 +0300
Subject: [PATCH 11/22] NFC: track the downstream implementation more closely
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 35 ++++++++++++--------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
2 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index e7d3a1e8bf172..815605099cd90 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -36,9 +36,9 @@ using namespace llvm;
namespace {
-class LowerIdxOps {
+class AMDGPULowerIdxOpsImpl {
public:
- LowerIdxOps(MachineFunction &MF)
+ AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
MRI(&MF.getRegInfo()) {}
@@ -52,7 +52,7 @@ class LowerIdxOps {
MachineRegisterInfo *MRI;
};
-void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -69,19 +69,21 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
- LoadMIB.addMemOperand(*MI.memoperands_begin());
+ auto *LoadMMO = *MI.memoperands_begin();
+ LoadMIB.addMemOperand(LoadMMO);
// Match what instruction selection does for a whole-dword access with a
// register index: record that the M0 write implied by the eventual movrel
// clobbers M0 (see AMDGPUAssignIdxToM0).
if (LoadIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+ Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
MachineOperand BitOffset = MI.getOperand(4);
if (BitOffset.isImm())
BitOffset.setImm(BitOffset.getImm() & 31);
- BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, DataReg)
.addReg(SrcAReg)
.add(BitOffset)
.add(MI.getOperand(3)); // bitsize
@@ -91,11 +93,15 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent();
auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+ const MCInstrDesc &II = TII->get(AMDGPU::V_BFI_B32_e64);
+
+ Register DataReg = StoreIdx.getDataOp().getReg();
+
Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
@@ -110,15 +116,16 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
// Synthesize a load MMO from the store's.
auto NewFlags = MachineMemOperand::MOLoad;
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
- LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
+ LoadMIB.addMemOperand(LoadMMO);
if (StoreIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
- BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
- .add(StoreIdx.getOperand(3)) // mask
- .addReg(StoreIdx.getDataOp().getReg())
- .addReg(SrcAReg);
+ auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
+ CoreMIB.add(StoreIdx.getOperand(3)); // mask
+ CoreMIB.addReg(DataReg);
+ CoreMIB.addReg(SrcAReg);
// Write the dword back.
auto StoreMIB =
@@ -135,7 +142,7 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-bool LowerIdxOps::run(MachineFunction &MF) {
+bool AMDGPULowerIdxOpsImpl::run(MachineFunction &MF) {
bool Changed = false;
LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
@@ -171,7 +178,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
// This is required lowering, not an optimization: nothing else expands the
// sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
// therefore must not be skipped for optnone functions.
- return LowerIdxOps(MF).run(MF);
+ return AMDGPULowerIdxOpsImpl(MF).run(MF);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -187,7 +194,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
PreservedAnalyses
AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!LowerIdxOps(MF).run(MF))
+ if (!AMDGPULowerIdxOpsImpl(MF).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b604a415b5713..7e10275726663 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19600,7 +19600,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
+ unsigned IndexOpIdx, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
>From b295c9540862de544e8b6ded86ba7a58ed4eb7be Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:16:13 +0300
Subject: [PATCH 12/22] Do not describe sub-dword VGPR-memory accesses by index
and width
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 347879b5f7e4f..0db28f589f5b8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -662,6 +662,13 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
}
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ // A sub-dword access is not described by a dword index and width alone -
+ // the bit position within the dword is part of the address - and has no
+ // entry in the width table. Report it as opaque so that callers such as
+ // areMemAccessesTriviallyDisjoint() fall back to assuming an overlap.
+ if (!AMDGPU::getVLdStIdxOpcodeInfoByOpcode(LdSt.getOpcode()))
+ return false;
+
BaseOp = &LdStIdx->getIdxOp();
OffsetOp = &LdStIdx->getOffsetOp();
>From bb1592fc0d5f4154239be540c8e86e77f5589974 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:47:57 +0300
Subject: [PATCH 13/22] Fix tests
---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 15 ++-
.../as-vgpr-index-demanded-bits.ll | 105 ++++++++++++++++++
2 files changed, 115 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 992b737009947..4910e9c56c4b7 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,13 +2,18 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
-; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
-; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
-; sub-dword code is the same there modulo register allocation, so only compile
-; and verify rather than checking the output a second time. What differs is the
-; waterfall around a divergent access, which uses a 64-bit exec mask.
+; Also compile for two further movrel subtargets. The expansion itself is
+; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
+; the sub-dword code is the same on both modulo register allocation, so only
+; compile and verify rather than checking the output again. gfx942 differs in
+; using a 64-bit exec mask for the waterfall around a divergent access, and
+; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
+; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
+; functions also cover the S_SET_VGPR_MSB encoding.
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 352b35d6aac49..be9465fb5edab 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- < %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- < %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
@@ -29,6 +31,28 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
%v = load i32, ptr addrspace(13) %ptr
@@ -47,6 +71,24 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-LABEL: store_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: store_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_endpgm
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
store i32 %val, ptr addrspace(13) %ptr
@@ -82,6 +124,37 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%m = and i32 %sum, 1023
%ptr = inttoptr i32 %m to ptr addrspace(13)
@@ -119,6 +192,37 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%o = or i32 %sum, 65536
%ptr = inttoptr i32 %o to ptr addrspace(13)
@@ -128,3 +232,4 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
+; GFX1250: {{.*}}
>From 9345cce2ad15f411a577b2f4608ccb4e2adc1fe5 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 18:14:45 +0300
Subject: [PATCH 14/22] Check the encodings of the sub-dword VGPR-memory
accesses
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 12 +---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 26 +++----
.../as-vgpr-bits-control-flow.ll | 7 +-
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 71 +++++++++++++++++++
.../as-vgpr-lower-idx-ops.mir | 16 ++---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 40 -----------
.../AddressSpaceVGPR/as-vgpr-verifier.mir | 32 +++++++++
7 files changed, 125 insertions(+), 79 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 815605099cd90..ec1463e0b2eb2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -15,7 +15,8 @@
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup, and before register allocation
+/// here take part in the usual M0 setup - they declare their write of M0
+/// themselves, like the pseudo they replace - and before register allocation
/// because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -71,11 +72,6 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
.add(LoadIdx.getOffsetOp());
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
- // Match what instruction selection does for a whole-dword access with a
- // register index: record that the M0 write implied by the eventual movrel
- // clobbers M0 (see AMDGPUAssignIdxToM0).
- if (LoadIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
@@ -118,8 +114,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
LoadMIB.addMemOperand(LoadMMO);
- if (StoreIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
@@ -134,8 +128,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
StoreMIB.addMemOperand(StoreMMO);
- if (StoreIdx.getIdxOp().isReg())
- StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0db28f589f5b8..39251e1fb1616 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5482,11 +5482,11 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
- // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
- // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
- // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
- // access has to record that the move clobbers M0, otherwise it could be
- // separated from the code that sets M0 up.
+ // The lowering of a VGPR "as memory" indexed access relies on it having
+ // exactly one memory operand, since a sub-dword store synthesizes a load
+ // operand from it, and on the index carrying no subregister, since
+ // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
+ // is declared by the instruction itself, so it needs no check here.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
@@ -5494,19 +5494,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
const MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg()) {
- if (IdxOp.getSubReg() != 0) {
- ErrInfo =
- "v_load/store_idx register index must not have a subregister.";
- return false;
- }
-
- if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
- !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
- ErrInfo = "v_load/store_idx with register index must have implicit-def "
- "$m0 on movrel subtargets.";
- return false;
- }
+ if (IdxOp.isReg() && IdxOp.getSubReg() != 0) {
+ ErrInfo = "v_load/store_idx register index must not have a subregister.";
+ return false;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 73edbbb7855e5..1f20bf28b2335 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -37,15 +37,16 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
new file mode 100644
index 0000000000000..32921a6244ebe
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -0,0 +1,71 @@
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+
+; The sub-dword accesses of the VGPR "as memory" address space (13) that
+; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
+; check assembly text, which does not show that the operand forms the expansion
+; picks can be assembled and encoded: a mask it builds from a known bit offset is
+; too wide for an inline constant and becomes a 32-bit literal, and a bit offset
+; reaches the bit-field extract as either an inline constant or an SGPR. On a
+; subtarget with more than 256 addressable VGPRs the whole-dword accesses the
+; expansion creates additionally carry the S_SET_VGPR_MSB encoding, so check
+; there too.
+;
+; The expansion runs after instruction selection and is handed the same pseudos
+; by both selectors, which produce identical code for these functions, so one
+; selector is enough here. as-vgpr-bits.ll covers the two separately.
+
+; A statically known bit offset turns the mask into a literal operand of the
+; bit-field insert.
+define amdgpu_ps void @store_i8_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_aligned>:
+; GFX1250: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i16_aligned>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xffff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i16
+ store i16 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; Without one the mask is computed at run time and is an SGPR instead.
+define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_dynamic_mask>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; Both bit offset and width of the extract are inline constants here.
+define amdgpu_ps i32 @load_i8_zext_aligned(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_zext_aligned>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_u32 v0, v0, 0, 8
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %z = zext i8 %x to i32
+ ret i32 %z
+}
+
+; A run-time bit offset is an SGPR, and the sign-extending load selects the
+; signed extract.
+define amdgpu_ps i32 @load_i8_sext_dynamic(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_sext_dynamic>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_i32 v0, v0, s0, 8
+ %x = load i8, ptr addrspace(13) %p
+ %s = sext i8 %x to i32
+ ret i32 %s
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c991990db933e..c8b4fefa3c3e3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -21,7 +21,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +39,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +57,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +78,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +100,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +122,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
deleted file mode 100644
index db32b6957abb3..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ /dev/null
@@ -1,40 +0,0 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-
-# Operand invariants of the VGPR "as memory" (address space 13) indexed
-# accesses, as enforced by SIInstrInfo::verifyInstruction.
-#
-# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
-# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
-# rewritten the index to M0, the access must record that the eventual move
-# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
-# The lowering also relies on the access having exactly one memory operand (it
-# synthesizes a load operand from it for a sub-dword store), and on the index
-# carrying no subregister, since it is rewritten in place to M0.
-
-# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
-# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
-# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
-
----
-name: operand_invariants
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
- %0:sreg_32 = COPY $sgpr0
- %1:vgpr_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr2_sgpr3
-
- ; No memory operand.
- %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
-
- ; Index with a subregister.
- %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
-
- ; Missing implicit-def $m0, for each of the four pseudo forms.
- %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
- V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
- V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
-...
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
new file mode 100644
index 0000000000000..d4e7ec49e1f40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -0,0 +1,32 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# The lowering relies on the access having exactly one memory operand, since a
+# sub-dword store synthesizes a load operand from it, and on the index carrying
+# no subregister, since AMDGPUAssignIdxToM0 rewrites it in place to M0. That the
+# access writes M0 is declared by the instructions themselves, so it needs no
+# check: MIR without the operand does not parse.
+
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
+
+---
+name: operand_invariants
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit-def $m0, implicit $exec
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit-def $m0, implicit $exec :: (load (s32), addrspace 13)
+
+ S_ENDPGM 0, implicit %3, implicit %4
+...
>From 28b9660de8861fe0a3f3ee8f62d0b211e7f500de Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 31 Jul 2026 20:15:02 +0300
Subject: [PATCH 15/22] Document the sub-dword access rules for the VGPR
address space
---
llvm/docs/AMDGPUUsage.rst | 7 +++++
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 26 ++++++++++---------
2 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index cb4d447d3766f..4840f2e3b6648 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1376,6 +1376,13 @@ supported for the ``amdgcn`` target.
of ``v3``. Storing 8 bytes to address ``32`` overwrites the contents of
``v[8:9]``.
+ An access narrower than a dword is implemented by extracting or inserting the
+ bits within the dword that contains it, so it must not straddle a dword
+ boundary: 8-bit accesses are unrestricted, while 16-bit accesses must be
+ 2-byte aligned. Wider accesses must be a whole number of dwords. Anything
+ else, such as an under-aligned 16-bit access, is reported as an error rather
+ than miscompiled.
+
Use of this address space by frontends is strongly discouraged. It has unusual
and subtle lifetime rules due to the potential for interaction with normal
register allocation. It exists primarily for internal purposes of the backend,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index db247727bac50..1b5807da5cb0e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1834,7 +1834,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
// unsupported access is diagnosed cleanly rather than failing to
// legalize.
- .customIf([=](const LegalityQuery &Query) -> bool {
+ .customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3552,7 +3552,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register BaseReg = PtrReg;
int64_t Offset = 0;
if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
- if (auto MaybeOff =
+ if (std::optional<ValueAndVReg> MaybeOff =
getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
BaseReg = PtrAdd->getBaseReg();
Offset = MaybeOff->Value.getSExtValue();
@@ -3565,7 +3565,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
HaveConstantBitOffset = true;
ConstantBitOffsetVal = 0;
} else {
- auto &VT = *Helper.getValueTracking();
+ GISelValueTracking &VT = *Helper.getValueTracking();
KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
if (BaseKB.isConstant()) {
Offset += BaseKB.getConstant().getZExtValue();
@@ -3575,18 +3575,18 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
}
// Setup common registers.
- const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
- auto Two = B.buildConstant(I32, 2);
- const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+ const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ MachineInstrBuilder Two = B.buildConstant(I32, 2);
+ const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
- const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ const MachineInstrBuilder BitWidthReg = B.buildConstant(I32, MemSize);
Register BitOffsetReg;
if (HaveConstantBitOffset) {
BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
} else {
// V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
// offset, so masking the high bits is unnecessary.
- auto Three = B.buildConstant(I32, 3);
+ MachineInstrBuilder Three = B.buildConstant(I32, 3);
BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
}
@@ -3600,19 +3600,21 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register ShiftAmt = BitOffsetReg;
if (!HaveConstantBitOffset) {
- auto Mask = B.buildConstant(I32, 31);
+ MachineInstrBuilder Mask = B.buildConstant(I32, 31);
ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
}
Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
- const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
- const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+ const MachineInstrBuilder MaskBase =
+ B.buildConstant(I32, (1u << MemSize) - 1);
+ const MachineInstrBuilder Mask = B.buildShl(I32, MaskBase, ShiftAmt);
B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
.addMemOperand(&MMO);
} else {
const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
- const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ const MachineInstrBuilder IsSExtReg =
+ B.buildConstant(I32, IsSExt ? 1 : 0);
Register Result =
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
{Index, BitWidthReg, BitOffsetReg, IsSExtReg})
>From 922919fe6fee853adf163ffcc0f21c4f3dea6fc4 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 3 Aug 2026 20:04:45 +0300
Subject: [PATCH 16/22] Adopt subarch triples in the sub-dword tests and drop
the redundant VALU
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../AddressSpaceVGPR/as-vgpr-bits-control-flow.ll | 8 ++++----
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 4 ++--
.../CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 12 ++++++------
.../AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir | 2 +-
6 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 89d7d74dac86b..485fd4cbcfb35 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1163,7 +1163,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
i32imm:$bitsize, VSrc_b32:$bitoffset,
i32imm:$is_sext_load)> {
let mayLoad = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1177,7 +1176,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
(ins VGPR_32:$data, SReg_32:$idx,
i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 1f20bf28b2335..23bb00886d179 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
; across control flow. A sub-dword store is a read-modify-write of the dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 32921a6244ebe..504e8034658cf 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.00-- -o - %s | llvm-mc -triple=amdgpu12.00-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -mtriple=amdgpu12.50-- -o - %s | llvm-mc -triple=amdgpu12.50-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
; The sub-dword accesses of the VGPR "as memory" address space (13) that
; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4910e9c56c4b7..9e7bea5d5f8cd 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for two further movrel subtargets. The expansion itself is
; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
@@ -10,10 +10,10 @@
; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
; functions also cover the S_SET_VGPR_MSB encoding.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c8b4fefa3c3e3..03646b899cbae 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.00-- -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
# AMDGPULowerIdxOps, in isolation:
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
index d4e7ec49e1f40..f42d75a904bfc 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgpu12.00-- -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
# Operand invariants of the VGPR "as memory" (address space 13) indexed
# accesses, as enforced by SIInstrInfo::verifyInstruction.
>From 7c3b8bf8035c9b51ae7c14b97dbad79d2bc1b514 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 15:22:07 +0300
Subject: [PATCH 17/22] Regenerate the sub-dword expectations for the M0 setup
change
---
.../as-vgpr-bits-control-flow.ll | 88 ++++++++-----------
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 79 +++++++----------
3 files changed, 67 insertions(+), 102 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 23bb00886d179..51f5e1fb468ff 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -22,31 +22,27 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -58,32 +54,29 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -107,10 +100,9 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
@@ -119,7 +111,6 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
@@ -136,20 +127,18 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, s3, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v2, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
@@ -257,16 +246,14 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
; GFX12-SDAG-NEXT: ; %bb.1: ; %then
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
@@ -285,19 +272,16 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
; GFX12-GISEL-NEXT: ; %bb.1: ; %then
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 504e8034658cf..5638ce53c394b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -42,7 +42,7 @@ define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %
define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
; DIS-LABEL: <store_i8_dynamic_mask>:
; DIS: v_movrels_b32_e32 v1, v0
-; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_bfi_b32 v0, s0, v0, v1
; DIS: v_movreld_b32_e32 v0, v0
%v = trunc i32 %vv to i8
store i8 %v, ptr addrspace(13) %p
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 9e7bea5d5f8cd..1c3f1db234aec 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -207,16 +207,14 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -229,19 +227,16 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
@@ -257,16 +252,14 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -279,19 +272,16 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i16 %v, ptr addrspace(13) %p
@@ -306,11 +296,8 @@ define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s0, s0, 2
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-NEXT: s_mov_b32 m0, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
; GFX12-NEXT: v_movreld_b32_e32 v0, v0
@@ -403,11 +390,10 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
@@ -444,7 +430,6 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
@@ -568,15 +553,13 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, s1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -589,25 +572,23 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 31
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_and_b32 s1, s0, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 0xff
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s1, s0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
>From 47249deb7d06c81428a530296e6917ea2d7fbc31 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 17:43:50 +0300
Subject: [PATCH 18/22] Inherit the M0 operand when expanding the sub-dword
VGPR-memory pseudos
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 25 ++++++++++++++++---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +--
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../as-vgpr-lower-idx-ops.mir | 23 +++++++++--------
4 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index ec1463e0b2eb2..ba632ce02074c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -14,10 +14,9 @@
/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
-/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup - they declare their write of M0
-/// themselves, like the pseudo they replace - and before register allocation
-/// because it introduces new virtual registers.
+/// This runs after SITargetLowering::finalizeLowering has given the pseudo its
+/// M0 operand, so the whole-dword accesses created here inherit it, and before
+/// register allocation because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -41,6 +40,7 @@ class AMDGPULowerIdxOpsImpl {
public:
AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ TRI(MF.getSubtarget<GCNSubtarget>().getRegisterInfo()),
MRI(&MF.getRegInfo()) {}
bool run(MachineFunction &MF);
@@ -48,11 +48,25 @@ class AMDGPULowerIdxOpsImpl {
private:
void lowerLoadIdxBits(MachineInstr &MI);
void lowerStoreIdxBits(MachineInstr &MI);
+ void inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const;
const SIInstrInfo *TII;
+ const SIRegisterInfo *TRI;
MachineRegisterInfo *MRI;
};
+// An indexed access takes its index from M0 where the subtarget has movrel, and
+// clobbers M0 through s_set_gpr_idx_on where it indexes with the VGPR indexing
+// mode. SITargetLowering::finalizeLowering picks between the two and records a
+// clobber as an implicit def, so a whole-dword access replacing the pseudo here
+// has to carry that over: the index itself comes along with the operand.
+void AMDGPULowerIdxOpsImpl::inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const {
+ if (MI.definesRegister(AMDGPU::M0, TRI))
+ MIB.addDef(AMDGPU::M0, RegState::Implicit);
+}
+
void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -70,6 +84,7 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
Register DataReg = LoadIdx.getDataOp().getReg();
@@ -106,6 +121,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
// The index is read again by the store below, so it does not die here.
LoadMIB->getOperand(1).setIsKill(false);
auto *StoreMMO = *MI.memoperands_begin();
@@ -127,6 +143,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.addReg(DstAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, StoreMIB);
StoreMIB.addMemOperand(StoreMMO);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 39251e1fb1616..ce8d8ea76bdab 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5485,8 +5485,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// The lowering of a VGPR "as memory" indexed access relies on it having
// exactly one memory operand, since a sub-dword store synthesizes a load
// operand from it, and on the index carrying no subregister, since
- // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
- // is declared by the instruction itself, so it needs no check here.
+ // SITargetLowering::finalizeLowering rewrites it in place to M0.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 485fd4cbcfb35..7096747e1ef48 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1165,7 +1165,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
let mayLoad = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Used for byte or short stores.
@@ -1178,7 +1177,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
let mayStore = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index 03646b899cbae..c5270bd219141 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -7,9 +7,10 @@
# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
#
-# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
-# virtual here and the implicit-def $m0 that instruction selection attached is
-# carried over to the whole-dword accesses the expansion creates.
+# The index and any M0 clobber are given to the pseudo beforehand, by
+# SITargetLowering::finalizeLowering, so the whole-dword accesses the expansion
+# creates carry both over: the index as their own index operand, and the clobber
+# as an implicit def where the subtarget indexes with the VGPR indexing mode.
---
name: lower_load_idx_bits_zext
@@ -21,7 +22,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +40,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +58,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +79,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +101,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +123,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
>From d0a16b6bf57de4a9b9199885ef4b7fc96e088843 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 14 Aug 2026 17:00:57 +0200
Subject: [PATCH 19/22] Regenerate the demanded-bits expectations for the
gfx1250 entry sequence
---
.../as-vgpr-index-demanded-bits.ll | 40 +++++++++++--------
1 file changed, 24 insertions(+), 16 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index be9465fb5edab..2327bf9fbe70a 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -34,9 +34,10 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
;
; GFX1250-SDAG-LABEL: load_masked_index:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -45,9 +46,10 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
;
; GFX1250-GISEL-LABEL: load_masked_index:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -74,18 +76,20 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
;
; GFX1250-SDAG-LABEL: store_masked_index:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: store_masked_index:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX1250-GISEL-NEXT: s_endpgm
@@ -127,9 +131,10 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
;
; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
@@ -142,9 +147,10 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
;
; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
@@ -195,9 +201,10 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
;
; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
@@ -210,9 +217,10 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
;
; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
>From c412e72df32214bcf991ed3e2f31cb90ebc418bb Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 13:29:37 -0400
Subject: [PATCH 20/22] Treat a sub-dword VGPR-memory load as divergent too
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +-
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 83 +++++++++++++++++++
2 files changed, 85 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 7e10275726663..8579edf0d9c46 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20945,9 +20945,10 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS ||
AS == AMDGPUAS::VGPR;
}
- // As above, after the pre-ISel combine. Without this a uniform index would
+ // As above, after the pre-ISel combine. Without these a uniform index would
// make the loaded value look uniform and consumers would v_readfirstlane it.
case AMDGPUISD::REG_LOAD:
+ case AMDGPUISD::REG_LOAD_BITS:
return true;
case ISD::CALLSEQ_END:
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index a69908541ff71..3f97ae34d6f2c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -295,6 +295,89 @@ define i32 @uniform_index_divergent_value(ptr addrspace(13) inreg %p, <4 x i32>
ret i32 %r
}
+; The same holds for a sub-dword access, which lowers to a different node. Its
+; value is per-lane too, so a uniform index does not make it uniform.
+define i32 @uniform_index_divergent_byte(ptr addrspace(13) inreg %p, <4 x i32> inreg %rsrc) {
+; GFX12-SDAG-LABEL: uniform_index_divergent_byte:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX12-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: uniform_index_divergent_byte:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: uniform_index_divergent_byte:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX942-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX942-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX942-SDAG-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX942-SDAG-NEXT: s_set_gpr_idx_off
+; GFX942-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX942-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX942-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX942-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX942-SDAG-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: uniform_index_divergent_byte:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX942-GISEL-NEXT: s_lshr_b32 s1, s0, 2
+; GFX942-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_on s1, gpr_idx(SRC0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_off
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX942-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX942-GISEL-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = load i8, ptr addrspace(13) %p, align 1
+ %z = zext i8 %v to i32
+ %r = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %rsrc, i32 %z, i32 0)
+ ret i32 %r
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
; GFX942: {{.*}}
>From 61be2c5c66d1e00f8558d17e3fff406778d5c4aa Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 26 Aug 2026 15:30:55 -0400
Subject: [PATCH 21/22] Pin the width guard that keeps sub-dword accesses out
of aliasing decisions
---
llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp | 70 +++++++++++++++++++
1 file changed, 70 insertions(+)
diff --git a/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
index a55b78a1de628..420fb50cb3463 100644
--- a/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
+++ b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
@@ -166,3 +166,73 @@ body: |
<< "M0 is redefined between these accesses, so their dword indices are "
"unrelated and they must not be reported disjoint";
}
+
+// A sub-dword access is not described by a dword index and width alone - the
+// bit position within the dword is part of the address - so these pseudos carry
+// no entry in the width table, deliberately. getMemOperandsWithOffsetWidth must
+// decline to describe one rather than reaching getBitWidth() and the
+// llvm_unreachable behind it. With assertions off that unreachable is UB, and
+// the caller it would reach through is an aliasing decision, so the failure
+// would be a garbage width silently declaring two overlapping accesses
+// disjoint.
+//
+// Note the class does match these pseudos, so the dyn_cast is not what protects
+// the call; the width-table lookup inside it is. The index here is a virtual
+// register so that the lookup is the only thing that can make an access opaque:
+// a physical index is declined separately, because it names no value to
+// compare.
+TEST_F(VGPRAsMemoryTest, SubDwordAccessHasNoDescribableWidth) {
+ StringRef MIRString = R"MIR(
+name: width
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $vgpr0
+
+ %0:sreg_32 = COPY $sgpr1
+ $vgpr1 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ $vgpr2 = V_LOAD_IDX_BITS %0, 0, 8, $sgpr0, 0, implicit $exec :: (load (s8), addrspace 13)
+ $vgpr3 = V_LOAD_IDX_BITS %0, 0, 8, $sgpr0, 0, implicit $exec :: (load (s8), addrspace 13)
+ S_ENDPGM 0
+...
+)MIR";
+
+ ASSERT_TRUE(parseMIR(MIRString));
+ MachineFunction &MF = getMF("width");
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const TargetRegisterInfo *TRI = ST.getRegisterInfo();
+ MachineBasicBlock *MBB = MF.getBlockNumbered(0);
+
+ auto Describe = [&](const MachineInstr &MI) {
+ SmallVector<const MachineOperand *, 4> BaseOps;
+ int64_t Offset = 0;
+ bool OffsetIsScalable = false;
+ LocationSize Width = LocationSize::precise(0);
+ return TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
+ OffsetIsScalable, Width, TRI);
+ };
+
+ const MachineInstr *FirstBits = nullptr;
+ const MachineInstr *SecondBits = nullptr;
+ for (MachineInstr &MI : *MBB) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_B32:
+ // A whole-dword access is describable: dword index plus a known width.
+ EXPECT_TRUE(Describe(MI));
+ break;
+ case AMDGPU::V_LOAD_IDX_BITS:
+ EXPECT_FALSE(Describe(MI))
+ << "a sub-dword access must be reported as opaque";
+ (FirstBits ? SecondBits : FirstBits) = &MI;
+ break;
+ default:
+ break;
+ }
+ }
+
+ // And the caller that would otherwise reach the unreachable: with no width to
+ // compare, two sub-dword accesses have to be assumed to overlap.
+ ASSERT_NE(FirstBits, nullptr);
+ ASSERT_NE(SecondBits, nullptr);
+ EXPECT_FALSE(TII->areMemAccessesTriviallyDisjoint(*FirstBits, *SecondBits));
+}
>From f0ca5a67243762206f99941a9507fe6e05328fd0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 9 Sep 2026 22:04:57 -0400
Subject: [PATCH 22/22] Set up M0 for sub-dword VGPR-memory accesses in the
custom inserter
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 10 +++-------
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 20 +++++++++----------
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 +++-----
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 14 +++++--------
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++----
llvm/lib/Target/AMDGPU/SIInstructions.td | 13 ++++++------
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 10 +++++-----
7 files changed, 35 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 1b5807da5cb0e..7f81d8a097896 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1830,10 +1830,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
})
- // VGPR ("as memory") extending loads are custom-lowered to
- // G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
- // unsupported access is diagnosed cleanly rather than failing to
- // legalize.
+ // Always take the custom path, so an unsupported access is
+ // diagnosed cleanly rather than failing to legalize.
.customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3540,9 +3538,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
- // Handle bytes and aligned shorts. These become a bit-field extract out of
- // the containing dword (loads), or a read-modify-write of it (stores); see
- // AMDGPULowerIdxOps.
+ // Bytes and aligned shorts; see AMDGPULowerIdxOps.
if (MemSize < 32) {
assert(MemSize == 8 || MemSize == 16);
assert(MemSize <= ValSize && ValSize <= 32);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index ba632ce02074c..85303d07da21e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -14,9 +14,9 @@
/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
-/// This runs after SITargetLowering::finalizeLowering has given the pseudo its
-/// M0 operand, so the whole-dword accesses created here inherit it, and before
-/// register allocation because it introduces new virtual registers.
+/// Runs after the custom inserter has given the pseudo its M0 operand, so the
+/// accesses created here inherit it, and before register allocation because it
+/// introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -56,11 +56,10 @@ class AMDGPULowerIdxOpsImpl {
MachineRegisterInfo *MRI;
};
-// An indexed access takes its index from M0 where the subtarget has movrel, and
-// clobbers M0 through s_set_gpr_idx_on where it indexes with the VGPR indexing
-// mode. SITargetLowering::finalizeLowering picks between the two and records a
-// clobber as an implicit def, so a whole-dword access replacing the pseudo here
-// has to carry that over: the index itself comes along with the operand.
+// Under the VGPR indexing mode the access clobbers M0 rather than reading it,
+// which the custom inserter records as an implicit def. A whole-dword access
+// replacing the pseudo has to carry that over; the index itself comes along
+// with the operand.
void AMDGPULowerIdxOpsImpl::inheritM0Def(const MachineInstr &MI,
const MachineInstrBuilder &MIB) const {
if (MI.definesRegister(AMDGPU::M0, TRI))
@@ -184,9 +183,8 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- // This is required lowering, not an optimization: nothing else expands the
- // sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
- // therefore must not be skipped for optnone functions.
+ // Required lowering, not an optimization: nothing else expands the
+ // sub-dword pseudos, so this must run even for optnone functions.
return AMDGPULowerIdxOpsImpl(MF).run(MF);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index bc2d80bbf675d..44b2373f564a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1839,9 +1839,8 @@ void GCNPassConfig::addFastRegAlloc() {
void GCNPassConfig::addPreRegAlloc() {
// Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
- // plus a bit-field extract/insert. The index of the access it expands has
- // already been copied into M0 by SITargetLowering::finalizeLowering, and the
- // whole-dword access created here inherits it.
+ // plus a bit-field extract/insert. Must follow the custom inserter, which
+ // puts the index in M0 for the access created here to inherit.
addPass(&AMDGPULowerIdxOpsID);
if (getOptLevel() != CodeGenOptLevel::None)
@@ -2654,8 +2653,7 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
- // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
- // plus a bit-field extract/insert; see the comment in GCNPassConfig.
+ // See GCNPassConfig::addPreRegAlloc for why this is unconditional.
addMachineFunctionPass(AMDGPULowerIdxOpsPass(), PMW);
if (getOptLevel() != CodeGenOptLevel::None)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8579edf0d9c46..bc35cbe8a6bc9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13609,16 +13609,13 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
return DAG.getErrorMergeValues(ResultTypes, MemOp->getChain(), DL);
};
- // Handle bytes and aligned shorts. These become a bit-field extract out of
- // the containing dword (loads), or a read-modify-write of it (stores); see
- // AMDGPULowerIdxOps.
+ // Bytes and aligned shorts; see AMDGPULowerIdxOps.
if (BitWidth < 32) {
if (BitWidth != 8 && BitWidth != 16)
return reportUnsupported();
- // The access becomes a bit-field extract from (or insert into) the dword
- // containing it, so it must not straddle a dword boundary. An 8-bit access
- // never can; a 16-bit one only if it is 2-byte aligned.
+ // A bit-field extract must not straddle a dword boundary, which natural
+ // alignment guarantees.
if (MemOp->getAlign() < Align(BitWidth / 8))
return reportUnsupported();
@@ -13721,9 +13718,8 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
return DAG.getMergeValues({Value, LoadChain}, DL);
}
- // Whole-dword accesses index by pointer >> 2, so an under-aligned one would
- // silently reach the containing dword. The sub-dword path above computes a
- // bit offset and carries its own alignment rule.
+ // Indexing by pointer >> 2 means an under-aligned access would silently reach
+ // the containing dword.
if (MemOp->getAlign() < Align(4))
return reportUnsupported();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ce8d8ea76bdab..16aebc3836b8c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5482,10 +5482,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
- // The lowering of a VGPR "as memory" indexed access relies on it having
- // exactly one memory operand, since a sub-dword store synthesizes a load
- // operand from it, and on the index carrying no subregister, since
- // SITargetLowering::finalizeLowering rewrites it in place to M0.
+ // Lowering relies on exactly one memory operand, since a sub-dword store
+ // synthesizes a load operand from it, and on the index carrying no
+ // subregister, since the custom inserter rewrites it in place to M0.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 7096747e1ef48..0c2f93593333a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1165,6 +1165,8 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
let mayLoad = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
+ // See the whole-dword pseudos above: the inserter supplies the M0 operand.
+ let usesCustomInserter = 1;
}
// Used for byte or short stores.
@@ -1177,11 +1179,11 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
let mayStore = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
+ // See the whole-dword pseudos above: the inserter supplies the M0 operand.
+ let usesCustomInserter = 1;
}
-// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
-// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
-// shape folds a constant dword offset into the pseudo's $offset operand.
+// As above, an (add idx, imm) shape folds a constant dword offset into $offset.
def : GCNPat<
(i32 (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
(i32 imm:$bitsize), i32:$bitoffset,
@@ -4965,9 +4967,8 @@ def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
let mayStore = 1;
}
-// Sub-dword (8/16-bit) counterparts of the above. In addition to the dword
-// index they carry the bit position of the access within that dword, and are
-// lowered to a read-modify-write of the whole dword (see AMDGPULowerIdxOps).
+// Sub-dword counterparts of the above, additionally carrying the bit position
+// within the dword. Lowered to a read-modify-write (see AMDGPULowerIdxOps).
def G_AMDGPU_REG_LOAD_BITS : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type1:$dword_index, type1:$bitsize,
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 1c3f1db234aec..7091c82d85419 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -326,13 +326,13 @@ define i32 @load_i8_divergent(ptr addrspace(13) %p) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
@@ -395,13 +395,13 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-SDAG-NEXT: ; %bb.2:
; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
More information about the llvm-branch-commits
mailing list