[llvm-branch-commits] [llvm] [AMDGPU] Add support for sub-dword types for address space 13 (PR #213223)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Sep 9 15:49:34 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/213223
>From 08b649f205b605fbe79bb941f1978589f5dc83bd Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 12:39:47 +0300
Subject: [PATCH 01/21] Add support for sub-dword types for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 10 +
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 2 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 161 ++++++-
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 201 ++++++++
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 9 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 6 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 18 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 11 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 148 +++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 18 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 82 ++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 449 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 69 +--
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
19 files changed, 1141 insertions(+), 71 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 86afda9e4bf49..b96af13cbef7b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -280,6 +280,9 @@ extern char &AMDGPURegBankLegalizeLegacyID;
void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
extern char &AMDGPUMarkLastScratchLoadID;
+void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
+extern char &AMDGPULowerIdxOpsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -494,6 +497,13 @@ class AMDGPUMarkLastScratchLoadPass
MachineFunctionAnalysisManager &AM);
};
+class AMDGPULowerIdxOpsPass
+ : public RequiredPassInfoMixin<AMDGPULowerIdxOpsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index dee02bbdeac34..5316b65725788 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -292,6 +292,8 @@ def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD_BITS, SIreg_load_bits>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE_BITS, SIreg_store_bits>;
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index f25570d352dd2..b2a09fb15cead 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -561,12 +561,20 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Only whole-dword
-// accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported
-// for now; sub-dword (8/16-bit) support lands later.
+// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
+// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
+// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
- return MemSize == ValSize &&
- AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+ if (MemSize == 8 || MemSize == 16) {
+ if (ValSize == MemSize)
+ return true;
+ if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
+ return true;
+ return false;
+ }
+ if (MemSize != ValSize)
+ return false;
+ return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
/// Return true if a load or store of the type should be lowered with a bitcast
@@ -1815,6 +1823,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
{S16, ConstantPtr, S8, GlobalAlign8}})
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
+ })
+ // VGPR ("as memory") extending loads are custom-lowered to
+ // G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
+ // unsupported access is diagnosed cleanly rather than failing to
+ // legalize.
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
if (ST.hasFlatAddressSpace()) {
@@ -3499,18 +3514,21 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
-/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into a legal
-/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE indexed by the pointer's dword offset
-/// (pointer >> 2). Parallels the SelectionDAG LowerLoadStoreVGPR.
+/// Lower a G_LOAD / G_STORE / G_SEXTLOAD / G_ZEXTLOAD on AMDGPUAS::VGPR into a
+/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
+/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
+/// Parallels the SelectionDAG LowerLoadStoreVGPR.
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
MachineMemOperand &MMO = **MI.memoperands_begin();
- const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ const unsigned Opcode = MI.getOpcode();
+ const bool IsStore = Opcode == AMDGPU::G_STORE;
Register ValReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
+ const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
const LLT ValTy = MRI.getType(ValReg);
const unsigned ValSize = ValTy.getSizeInBits();
// The GISel selection patterns for the indexed pseudos - and for the shift /
@@ -3519,23 +3537,24 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Only dword-aligned whole-dword, non-extending/non-truncating accesses are
- // implemented. Reject anything else with a diagnostic instead of failing to
- // legalize (sub-dword support lands in a later change).
+ // Dword-aligned whole-dword and 8-/16-bit accesses are implemented. Reject
+ // anything else with a diagnostic instead of failing to legalize.
//
- // The alignment is checked here rather than in the size predicate: the index
- // built below is the pointer shifted right by two, which discards the low two
- // bits rather than accounting for them, so an under-aligned access would
- // silently reach the dword containing the address instead of the bytes asked
- // for. That is a property of how the address is formed, not of the size.
- if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
- ValSize) ||
- MMO.getAlign() < Align(4)) {
+ // The alignment is checked here rather than in the size predicate, since it
+ // is a property of how the address is formed rather than of the size: a
+ // whole-dword access addresses registers by the dword index pointer >> 2,
+ // which discards the low two bits rather than accounting for them, so an
+ // under-aligned one would silently reach the dword containing the address.
+ // The sub-dword path below computes a bit offset instead, and carries its own
+ // alignment rule.
+ if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize) ||
+ (MemSize >= 32 && MMO.getAlign() < Align(4))) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword and 8-/16-bit loads and stores are "
+ "implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
@@ -3543,6 +3562,102 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (MemSize < 32) {
+ assert(MemSize == 8 || MemSize == 16);
+ assert(MemSize <= ValSize && ValSize <= 32);
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // known constant.
+ Register BaseReg = PtrReg;
+ int64_t Offset = 0;
+ if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
+ if (auto MaybeOff =
+ getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
+ BaseReg = PtrAdd->getBaseReg();
+ Offset = MaybeOff->Value.getSExtValue();
+ }
+ }
+
+ bool HaveConstantBitOffset = false;
+ int64_t ConstantBitOffsetVal = 0;
+ if (Offset == 0 && MMO.getAlign() >= Align(4)) {
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = 0;
+ } else {
+ auto &VT = *Helper.getValueTracking();
+ KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = (Offset & 3) * 8;
+ }
+ }
+
+ // Setup common registers.
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ Register BitOffsetReg;
+ if (HaveConstantBitOffset) {
+ BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
+ // offset, so masking the high bits is unnecessary.
+ auto Three = B.buildConstant(I32, 3);
+ BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
+ }
+
+ if (IsStore) {
+ LLT IValTy = LLT::integer(ValSize);
+ Register Value = ValReg;
+ if (IValTy != ValTy)
+ Value = B.buildBitcast(IValTy, Value).getReg(0);
+ if (ValSize != 32)
+ Value = B.buildAnyExt(I32, Value).getReg(0);
+
+ Register ShiftAmt = BitOffsetReg;
+ if (!HaveConstantBitOffset) {
+ auto Mask = B.buildConstant(I32, 31);
+ ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
+ }
+ Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
+
+ const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
+ const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
+ .addMemOperand(&MMO);
+ } else {
+ const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
+ const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ Register Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
+ {Index, BitWidthReg, BitOffsetReg, IsSExtReg})
+ .addMemOperand(&MMO)
+ .getReg(0);
+
+ if (ValTy != I32) {
+ LLT ResultTy = I32;
+ if (ValSize != 32) {
+ ResultTy = LLT::integer(ValSize);
+ Result = B.buildTrunc(ResultTy, Result).getReg(0);
+ }
+ if (ResultTy != ValTy)
+ Result = B.buildBitcast(ValTy, Result).getReg(0);
+ }
+
+ B.buildCopy(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+ }
+
const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
MachineInstrBuilder Two = B.buildConstant(I32, 2);
const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
@@ -3585,7 +3700,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
- if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ // G_LOAD as well as the extending loads (G_SEXTLOAD / G_ZEXTLOAD) are
+ // custom-lowered; the latter only exist for sub-dword accesses.
+ if (AddrSpace == AMDGPUAS::VGPR)
return lowerLoadStoreVGPR(Helper, MI);
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
new file mode 100644
index 0000000000000..95cfe42df7e6d
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -0,0 +1,201 @@
+//===- AMDGPULowerIdxOps.cpp - Expand sub-dword VGPR-memory accesses -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Expand the sub-dword VGPR "as memory" (address space 13) pseudos into a
+/// whole-dword indexed access plus a bit-field extract or insert:
+///
+/// V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+///
+/// A sub-dword store is therefore a read-modify-write of the containing dword.
+/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
+/// here take part in the usual M0 setup, and before register allocation
+/// because it introduces new virtual registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-lower-idx-ops"
+
+namespace {
+
+class LowerIdxOps {
+public:
+ LowerIdxOps(MachineFunction &MF)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ MRI(&MF.getRegInfo()) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ void lowerLoadIdxBits(MachineInstr &MI);
+ void lowerStoreIdxBits(MachineInstr &MI);
+
+ const SIInstrInfo *TII;
+ MachineRegisterInfo *MRI;
+};
+
+void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
+
+ const bool IsSigned = MI.getOperand(5).getImm() != 0;
+ const MCInstrDesc &II =
+ TII->get(IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPUMI::VLoadIdxInst::getOpcodeForBitWidth(32)),
+ SrcAReg)
+ .add(LoadIdx.getIdxOp())
+ .add(LoadIdx.getOffsetOp());
+ LoadMIB.addMemOperand(*MI.memoperands_begin());
+ // Match what instruction selection does for a whole-dword access with a
+ // register index: record that the M0 write implied by the eventual movrel
+ // clobbers M0 (see AMDGPUAssignIdxToM0).
+ if (LoadIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Extract the accessed bits out of it.
+ MachineOperand BitOffset = MI.getOperand(4);
+ if (BitOffset.isImm())
+ BitOffset.setImm(BitOffset.getImm() & 31);
+
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ .addReg(SrcAReg)
+ .add(BitOffset)
+ .add(MI.getOperand(3)); // bitsize
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineFunction *MF = MBB->getParent();
+ auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB = BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ // The index is read again by the store below, so it does not die here.
+ LoadMIB->getOperand(1).setIsKill(false);
+ auto *StoreMMO = *MI.memoperands_begin();
+ // Synthesize a load MMO from the store's.
+ auto NewFlags = MachineMemOperand::MOLoad;
+ NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
+ LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ if (StoreIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Insert the stored bits into it.
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
+ .add(StoreIdx.getOperand(3)) // mask
+ .addReg(StoreIdx.getDataOp().getReg())
+ .addReg(SrcAReg);
+
+ // Write the dword back.
+ auto StoreMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_STORE_IDX_B32))
+ .addReg(DstAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ StoreMIB.addMemOperand(StoreMMO);
+ if (StoreIdx.getIdxOp().isReg())
+ StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+bool LowerIdxOps::run(MachineFunction &MF) {
+ bool Changed = false;
+
+ LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (auto MII = MBB.begin(), E = MBB.end(); MII != E;) {
+ MachineInstr &MI = *MII++;
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_BITS:
+ lowerLoadIdxBits(MI);
+ Changed = true;
+ break;
+ case AMDGPU::V_STORE_IDX_BITS:
+ lowerStoreIdxBits(MI);
+ Changed = true;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ return Changed;
+}
+
+class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return LowerIdxOps(MF).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Lower Idx Ops"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!LowerIdxOps(MF).run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPULowerIdxOpsLegacy::ID = 0;
+
+char &llvm::AMDGPULowerIdxOpsID = AMDGPULowerIdxOpsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPULowerIdxOpsLegacy, DEBUG_TYPE, "AMDGPU Lower Idx Ops",
+ false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
index 697c63e2079d1..970e9916f14dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -19,14 +19,19 @@ using namespace llvm;
using namespace AMDGPUMI;
unsigned VLoadStoreIdxInst::getBitWidth() const {
+ unsigned Opc = getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ llvm_unreachable("V_LOAD/STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
if (!Info)
llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
return Info->BitWidth;
}
int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_LOAD_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
if (!Info)
@@ -41,6 +46,8 @@ unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
}
int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
if (!Info)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index 24e24aab53152..e99594062871a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -37,7 +37,10 @@ class VLoadStoreIdxInst : public MachineInstr {
unsigned getBitWidth() const;
static bool classof(const MachineInstr *MI) {
- return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc) != nullptr;
}
};
@@ -47,8 +50,11 @@ class VLoadIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && !Info->IsStore;
}
};
@@ -59,8 +65,11 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && Info->IsStore;
}
};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 546a864863991..a89d4422eeaaa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -120,6 +120,7 @@ MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-lower-idx-ops", AMDGPULowerIdxOpsPass())
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index b3daf98ed5511..e84c16950a9ba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1399,6 +1399,12 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForGOpcs({G_AMDGPU_REG_STORE})
.Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD_BITS})
+ .Any({{BRC}, {{VgprBRC}, {Sgpr32_WF, Sgpr32, Vgpr32, Sgpr32}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE_BITS})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF, Vgpr32}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 5af7af4ef14be..253215e70dc45 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3089,7 +3089,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// The dword index (operand 1) must be uniform; a divergent index needs a
// waterfall loop.
applyDefaultMapping(OpdMapper);
@@ -4496,10 +4498,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
break;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// data/result is a VGPR value; the dword index is uniform (SGPR).
OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ const unsigned Opc = MI.getOpcode();
+ if (Opc == AMDGPU::G_AMDGPU_REG_LOAD_BITS) {
+ // {dst, index, bitsize, bitoffset, is_sext_load}
+ OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+ OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+ } else if (Opc == AMDGPU::G_AMDGPU_REG_STORE_BITS) {
+ // {value, index, mask}
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ }
break;
}
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f25fab4b24721..f2675ebda1206 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -695,6 +695,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPURegBankLegalizeLegacyPass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
+ initializeAMDGPULowerIdxOpsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1837,6 +1838,12 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert. The index of the access it expands has
+ // already been copied into M0 by SITargetLowering::finalizeLowering, and the
+ // whole-dword access created here inherits it.
+ addPass(&AMDGPULowerIdxOpsID);
+
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
@@ -2647,6 +2654,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert; see the comment in GCNPassConfig.
+ addMachineFunctionPass(AMDGPULowerIdxOpsPass(), PMW);
+
if (getOptLevel() != CodeGenOptLevel::None)
addMachineFunctionPass(AMDGPUPrepareAGPRAllocPass(), PMW);
if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b18ca6bd24040..5c9f8a2063847 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -90,6 +90,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUMemoryUtils.cpp
AMDGPUCoExecSchedStrategy.cpp
AMDGPUIGroupLP.cpp
+ AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5a5d3ec0ebf2e..cada6ed509a9e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13495,17 +13495,18 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Only dword-aligned whole-dword, non-extending/non-truncating accesses are
- // implemented. Reject anything else with a diagnostic (replacing the value
- // with poison) instead of failing instruction selection. Both callers -
- // operation legalization and the pre-ISel combine - replace the node with
- // this result, so the diagnostic is emitted exactly once.
+ // Dword-aligned whole-dword and 8-/16-bit accesses are implemented. Reject
+ // anything else with a diagnostic (replacing the value with poison) instead
+ // of failing instruction selection. Both callers - operation legalization and
+ // the pre-ISel combine - replace the node with this result, so the diagnostic
+ // is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword loads and stores are implemented",
+ "dword-aligned whole-dword and 8-/16-bit loads and stores are "
+ "implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13513,15 +13514,120 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
{DAG.getPOISON(Op.getValueType()), MemOp->getChain()}, DL);
};
- if (BitWidth < 32)
- return reportUnsupported();
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (BitWidth < 32) {
+ if (BitWidth != 8 && BitWidth != 16)
+ return reportUnsupported();
+
+ // Bail out for sub-dword types we cannot handle.
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
+ Load->getValueType(0).getSizeInBits() > 32)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore() &&
+ Store->getValue().getValueType() != MVT::i16 &&
+ Store->getValue().getValueType() != MVT::i32)
+ return reportUnsupported();
+ }
+
+ SDValue Ptr = MemOp->getBasePtr();
+
+ // Split the address into base and constant offset.
+ SDValue Base;
+ int32_t Offset;
+ if (DAG.isBaseWithConstantOffset(Ptr)) {
+ Base = Ptr.getOperand(0);
+ Offset = cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue();
+ } else if (auto *C = dyn_cast<ConstantSDNode>(Ptr)) {
+ Base = DAG.getConstant(0, DL, MVT::i32);
+ Offset = C->getSExtValue();
+ } else {
+ Base = Ptr;
+ Offset = 0;
+ }
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // constant.
+ KnownBits BaseKB;
+ if (Offset == 0 && MemOp->getAlign() >= Align(4)) {
+ BaseKB = KnownBits::makeConstant(APInt::getZero(2));
+ } else {
+ BaseKB = DAG.computeKnownBits(Base).trunc(2);
+ }
- // The dword index built below is the pointer shifted right by two, which
- // discards the low two bits rather than accounting for them, so an
- // under-aligned access would silently reach the dword containing the address
- // instead of the bytes asked for.
+ SDValue BitOffset;
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ BitOffset = DAG.getConstant((Offset & 3) * 8, DL, MVT::i32);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the least 5 bits of the bit
+ // offset, so we do not have to mask off the high bits.
+ BitOffset = DAG.getNode(ISD::SHL, DL, MVT::i32, Ptr,
+ DAG.getConstant(3, DL, MVT::i32));
+ }
+
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
+ DAG.getConstant(2, DL, MVT::i32));
+ SDValue BitSizeImm = DAG.getConstant(BitWidth, DL, MVT::i32);
+ SDValue Chain = MemOp->getChain();
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ EVT ValVT = Value.getValueType();
+ if (!ValVT.isScalarInteger())
+ Value = DAG.getNode(
+ ISD::BITCAST, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ValVT.getSizeInBits()), Value);
+ Value = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Value);
+
+ // ISD::SHL produces poison for shift amounts >= bit width, which generic
+ // DAG combines rely on. We need the mask for correctness, but it should
+ // get optimized away during final instruction selection.
+ SDValue ShiftOffset = DAG.getNode(ISD::AND, DL, MVT::i32, BitOffset,
+ DAG.getConstant(31, DL, MVT::i32));
+ Value = DAG.getNode(ISD::SHL, DL, MVT::i32, Value, ShiftOffset);
+
+ SDValue MaskBase = DAG.getConstant((1u << BitWidth) - 1, DL, MVT::i32);
+ SDValue Mask = DAG.getNode(ISD::SHL, DL, MVT::i32, MaskBase, ShiftOffset);
+
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE_BITS, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index, Mask}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ bool IsSExt = LoadOp->getExtensionType() == ISD::SEXTLOAD;
+ SDValue IsSExtImm = DAG.getConstant(IsSExt ? 1 : 0, DL, MVT::i32);
+ SDValue Value = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD_BITS, DL, DAG.getVTList(MVT::i32, MVT::Other),
+ {Chain, Index, BitSizeImm, BitOffset, IsSExtImm}, MemVT,
+ LoadOp->getMemOperand());
+ EVT ResVT = LoadOp->getValueType(0);
+ if (ResVT == MVT::i32)
+ return Value;
+
+ SDValue LoadChain = Value.getValue(1);
+ unsigned ResWidth = ResVT.getSizeInBits();
+ if (ResWidth < 32)
+ Value =
+ DAG.getNode(ISD::TRUNCATE, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ResWidth), Value);
+ if (Value.getValueType() != ResVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
+ return DAG.getMergeValues({Value, LoadChain}, DL);
+ }
+ // Everything below is a whole-dword access. It addresses registers by the
+ // dword index pointer >> 2, which discards the low two bits rather than
+ // accounting for them, so an under-aligned one would silently reach the dword
+ // containing the address instead of the bytes asked for. The sub-dword path
+ // above computes a bit offset instead and carries its own alignment rule.
if (MemOp->getAlign() < Align(4))
return reportUnsupported();
+
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
@@ -19387,7 +19493,9 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
case AMDGPUISD::REG_LOAD:
- case AMDGPUISD::REG_STORE: {
+ case AMDGPUISD::REG_STORE:
+ case AMDGPUISD::REG_LOAD_BITS:
+ case AMDGPUISD::REG_STORE_BITS: {
const SIMachineFunctionInfo *MFI =
DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
unsigned NumAddressableVGPRs =
@@ -19395,7 +19503,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0;
+ unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
@@ -19403,10 +19511,22 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case AMDGPUISD::REG_STORE:
IndexOpIdx = 2;
break;
+ case AMDGPUISD::REG_LOAD_BITS:
+ IndexOpIdx = 1;
+ BitOffOpIdx = 3;
+ break;
+ case AMDGPUISD::REG_STORE_BITS:
+ IndexOpIdx = 2;
+ break;
}
if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
return SDValue(N, 0);
+ if (BitOffOpIdx) {
+ APInt BitOffMask = APInt::getLowBitsSet(32, 5);
+ if (SimplifyDemandedBits(N->getOperand(BitOffOpIdx), BitOffMask, DCI))
+ return SDValue(N, 0);
+ }
break;
}
case ISD::LOAD: {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d0d6e509c13c5..92426ae8e29b1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11264,7 +11264,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
// A VGPR ("as memory") indexed load is always divergent: it reads the wave's
// per-lane view of its vector registers, so even a uniform index yields a
// per-lane (divergent) value.
- if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD ||
+ Opcode == AMDGPU::G_AMDGPU_REG_LOAD_BITS)
return ValueUniformity::NeverUniform;
// Loads from the private and flat address spaces are divergent, because
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 7bc293b060820..bd29cf0064da9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -70,6 +70,24 @@ def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+// VGPR address space sub-dword (8/16-bit) load/store with dword index. The
+// access is described by the dword index plus the bit position within that
+// dword; the store form carries a pre-shifted value and an insert mask.
+def SDTRegIdxBitsLoad : SDTypeProfile<1, 4,
+ [SDTCisInt<0>, // result (i32 or i16)
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>, // bitsize
+ SDTCisVT<3, i32>, // bitoffset
+ SDTCisVT<4, i32>]>; // is_sext_load
+def SDTRegIdxBitsStore : SDTypeProfile<0, 3,
+ [SDTCisInt<0>, // data (i32 or i16), pre-shifted insert value
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>]>; // insert mask
+def SIreg_load_bits : SDNode<"AMDGPUISD::REG_LOAD_BITS", SDTRegIdxBitsLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store_bits : SDNode<"AMDGPUISD::REG_STORE_BITS", SDTRegIdxBitsStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index da083b54b8b7c..7a9790221f290 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1166,6 +1166,70 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+// Used for byte or short loads.
+// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
+// sign-extended loads.
+//
+// Only the least 5 bits of $bitoffset are significant, the rest are ignored.
+// Only offset values of 0, 8, 16, or 24 are meaningful (after masking to the
+// 5 LSBs).
+//
+// Only bitsize of 8 or 16 bits are supported and must be a compile-time
+// constant.
+class V_LOAD_IDX_BITS<RegisterClass data_rc>
+ : VPseudoInstSI<(outs data_rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
+ VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
+
+// Used for byte or short stores.
+//
+// $data holds the pre-shifted insert value and $mask holds the insert mask
+// that is used when lowering to V_BFI_B32.
+class V_STORE_IDX_BITS<RegisterClass data_op_rc>
+ : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
+
+// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
+// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
+// shape folds a constant dword offset into the pseudo's $offset operand.
+multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
+ Instruction store_inst, ValueType vt> {
+ def : GCNPat<
+ (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+ def : GCNPat<
+ (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (store_inst $data, $idx, imm:$offset, $mask)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
+ (store_inst $data, $idx, 0, $mask)>;
+}
+
+defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4933,6 +4997,24 @@ def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
let mayStore = 1;
}
+// Sub-dword (8/16-bit) counterparts of the above. In addition to the dword
+// index they carry the bit position of the access within that dword, and are
+// lowered to a read-modify-write of the whole dword (see AMDGPULowerIdxOps).
+def G_AMDGPU_REG_LOAD_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index, type1:$bitsize,
+ type1:$bitoffset, type1:$is_sext_load);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index, type1:$mask);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
new file mode 100644
index 0000000000000..31c9b44e7236d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -0,0 +1,449 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
+; address space (13). A load reads the containing dword with an M0-relative
+; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
+; read-modify-write of that dword using v_bfi_b32 (see AMDGPULowerIdxOps).
+
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+define i32 @load_i8_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i8_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = sext i8 %x to i32
+ ret i32 %y
+}
+
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+define i32 @load_i16_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i16_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = sext i16 %x to i32
+ ret i32 %y
+}
+
+; A dword-aligned access has a statically known bit offset of zero.
+define i32 @load_i8_aligned(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+; A constant byte offset folds into a constant bit offset.
+define i32 @load_i8_const_offset(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i8_const_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 6
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_const_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 6
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 6
+ %x = load i8, ptr addrspace(13) %q, align 2
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+; GFX12-SDAG-LABEL: store_i16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-LABEL: store_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; A divergent index is made uniform by a waterfall loop.
+define i32 @load_i8_divergent(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 31, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-GISEL-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 0cbda8e8ff646..28dcb1b27f86e 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,49 +1,54 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are not implemented
-; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
-; rather than failing with "cannot select" / "unable to legalize" - or, worse,
-; silently generating wrong code.
-
-; Sub-dword (8/16-bit) accesses are not yet implemented; support lands in a
-; later change.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; Accesses of the VGPR "as memory" address space (13) that are still not
+; implemented must be rejected with a clean diagnostic on both SelectionDAG and
+; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
+; Dword-aligned whole-dword and 8-/16-bit accesses are implemented; see
+; as-vgpr-basic.ll and
+; as-vgpr-bits.ll.
+
+; A sub-dword load extended into a value wider than a dword.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
- ret i8 %x
+ %y = zext i8 %x to i64
+ ret i64 %y
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define i16 @load_i16(ptr addrspace(13) inreg %p) {
- %x = load i16, ptr addrspace(13) %p
- ret i16 %x
+; A memory size that is neither a whole dword nor 8/16 bits.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define i1 @load_i1(ptr addrspace(13) inreg %p) {
+ %x = load i1, ptr addrspace(13) %p
+ ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
- store i8 %v, ptr addrspace(13) %p
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
+ store i1 %v, ptr addrspace(13) %p
ret void
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
-define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
- store i16 %v, ptr addrspace(13) %p
- ret void
+; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
+ %x = load <14 x i32>, ptr addrspace(13) %p
+ ret <14 x i32> %x
}
-; An access addresses registers by the dword index pointer >> 2, which discards
-; the low two bits rather than accounting for them. An under-aligned one would
-; therefore reach the dword containing the address instead of the bytes asked
-; for - the same code as a correctly aligned access, reading the wrong data with
-; nothing to show for it.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+; A whole-dword access addresses registers by the dword index pointer >> 2,
+; which discards the low two bits rather than accounting for them. An
+; under-aligned one would therefore reach the dword containing the address
+; instead of the bytes asked for - the same code as a correctly aligned access,
+; reading the wrong data with nothing to show for it. A sub-dword access
+; computes a bit offset instead, and is held only to its own natural alignment.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
%x = load i32, ptr addrspace(13) %p, align 1
ret i32 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
store i32 %v, ptr addrspace(13) %p, align 1
ret void
@@ -56,3 +61,11 @@ define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
%x = load i64, ptr addrspace(13) %p, align 4
ret i64 %x
}
+
+; And a byte access at align 1 is fine, which is what the whole-dword-only
+; restriction must not catch.
+; CHECK-NOT: in function load_i8_align1
+define i8 @load_i8_align1(ptr addrspace(13) inreg %p) {
+ %x = load i8, ptr addrspace(13) %p, align 1
+ ret i8 %x
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 89f72e559ba52..ba683ded7e575 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,6 +68,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
; GCN-O0-NEXT: two-address-instruction
@@ -215,6 +216,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-lower-idx-ops
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
; GCN-O2-NEXT: dead-mi-elimination
@@ -402,6 +404,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-lower-idx-ops
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
; GCN-O3-NEXT: dead-mi-elimination
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index bb7ed3b58f8af..19d8cbae5c1e8 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
; GCN-O0-NEXT: Two-Address instruction pass
@@ -354,6 +355,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
@@ -683,6 +685,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -1016,6 +1019,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Lower Idx Ops
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1365,6 +1369,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Lower Idx Ops
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
>From 67ea48cc9bd9c0f27cffdaa873391186e9bb52df Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:14:32 +0300
Subject: [PATCH 02/21] Correctly diagnose the mis-aligned sub-dowrd accesses
instead of miscompiling them
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 45 ++++++----
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 20 +++--
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 43 +++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 88 +++++++++++--------
4 files changed, 134 insertions(+), 62 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index b2a09fb15cead..62c3264f9f7ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -561,11 +561,26 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
-// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
-// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
-static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+// memory access producing/consuming a ValSize-bit value at the given alignment.
+// Whole-dword accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo)
+// are supported when dword aligned, as are 8-/16-bit accesses, including
+// extending loads into a 16- or 32-bit value.
+//
+// A sub-dword access is implemented as a bit-field extract from (or insert
+// into) the dword containing it, so it must not straddle a dword boundary. An
+// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
+// natural alignment covers both, and is what lets the bit offset within the
+// dword be computed from a possibly dynamic pointer.
+//
+// A whole-dword access addresses registers by the dword index pointer >> 2,
+// which discards the low two bits rather than accounting for them, so an
+// under-aligned one would silently access the containing dword instead of the
+// bytes asked for.
+static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment) {
if (MemSize == 8 || MemSize == 16) {
+ if (Alignment < Align(MemSize / 8))
+ return false;
if (ValSize == MemSize)
return true;
if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
@@ -574,6 +589,8 @@ static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
}
if (MemSize != ValSize)
return false;
+ if (Alignment < Align(4))
+ return false;
return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
@@ -3537,24 +3554,16 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Dword-aligned whole-dword and 8-/16-bit accesses are implemented. Reject
- // anything else with a diagnostic instead of failing to legalize.
- //
- // The alignment is checked here rather than in the size predicate, since it
- // is a property of how the address is formed rather than of the size: a
- // whole-dword access addresses registers by the dword index pointer >> 2,
- // which discards the low two bits rather than accounting for them, so an
- // under-aligned one would silently reach the dword containing the address.
- // The sub-dword path below computes a bit offset instead, and carries its own
- // alignment rule.
- if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize) ||
- (MemSize >= 32 && MMO.getAlign() < Align(4))) {
+ // Dword-aligned whole-dword and naturally aligned 8-/16-bit accesses are
+ // implemented. Reject anything else with a diagnostic instead of failing to
+ // legalize.
+ if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword and 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index cada6ed509a9e..3c12064a088a2 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13495,18 +13495,18 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Dword-aligned whole-dword and 8-/16-bit accesses are implemented. Reject
- // anything else with a diagnostic (replacing the value with poison) instead
- // of failing instruction selection. Both callers - operation legalization and
- // the pre-ISel combine - replace the node with this result, so the diagnostic
- // is emitted exactly once.
+ // Dword-aligned whole-dword and naturally aligned 8-/16-bit accesses are
+ // implemented. Reject anything else with a diagnostic (replacing the value
+ // with poison) instead of failing instruction selection. Both callers -
+ // operation legalization and the pre-ISel combine - replace the node with
+ // this result, so the diagnostic is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "dword-aligned whole-dword and 8-/16-bit loads and stores are "
- "implemented",
+ "dword-aligned whole-dword and naturally aligned 8-/16-bit loads and "
+ "stores are implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13521,6 +13521,12 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
if (BitWidth != 8 && BitWidth != 16)
return reportUnsupported();
+ // The access becomes a bit-field extract from (or insert into) the dword
+ // containing it, so it must not straddle a dword boundary. An 8-bit access
+ // never can; a 16-bit one only if it is 2-byte aligned.
+ if (MemOp->getAlign() < Align(BitWidth / 8))
+ return reportUnsupported();
+
// Bail out for sub-dword types we cannot handle.
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 31c9b44e7236d..4b1703c1b6043 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -447,3 +447,46 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
store i8 %v, ptr addrspace(13) %p
ret void
}
+
+; A 2-byte aligned 16-bit access sits at bit offset 0 or 16, so it always fits
+; within the containing dword even though the offset is not known statically.
+; (An under-aligned 16-bit access is rejected; see as-vgpr-unsupported.ll.)
+define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i16_offset2:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i16_offset2:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 2
+ %x = load i16, ptr addrspace(13) %q, align 2
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 28dcb1b27f86e..22dc236753367 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,15 +1,55 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are still not
-; implemented must be rejected with a clean diagnostic on both SelectionDAG and
-; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
-; Dword-aligned whole-dword and 8-/16-bit accesses are implemented; see
-; as-vgpr-basic.ll and
+; Accesses of the VGPR "as memory" address space (13) that are not implemented
+; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
+; rather than failing with "cannot select" / "unable to legalize" - or, worse,
+; silently generating wrong code. Dword-aligned whole-dword and naturally
+; aligned 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
; as-vgpr-bits.ll.
+; A whole-dword access addresses registers by the dword index pointer >> 2,
+; which discards the low two bits rather than accounting for them. An
+; under-aligned one would therefore access the dword containing the address
+; instead of the bytes asked for, reading or writing the wrong data with
+; nothing to show for it, so it has to be rejected here.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
+ %x = load i32, ptr addrspace(13) %p, align 1
+ ret i32 %x
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
+ store i32 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
+; Alignment is required of the pointer, not of the accessed type: a 64-bit
+; access needs only the dword alignment the index computation relies on.
+; CHECK-NOT: in function load_i64_align4
+define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
+ %x = load i64, ptr addrspace(13) %p, align 4
+ ret i64 %x
+}
+
+; An under-aligned 16-bit access may straddle a dword boundary, which the
+; bit-field extract / insert lowering cannot express.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i16_align1(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p, align 1
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i16_align1(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
+
; A sub-dword load extended into a value wider than a dword.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
%y = zext i8 %x to i64
@@ -17,53 +57,27 @@ define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
}
; A memory size that is neither a whole dword nor 8/16 bits.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i1 @load_i1(ptr addrspace(13) inreg %p) {
%x = load i1, ptr addrspace(13) %p
ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
store i1 %v, ptr addrspace(13) %p
ret void
}
; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
%x = load <14 x i32>, ptr addrspace(13) %p
ret <14 x i32> %x
}
-; A whole-dword access addresses registers by the dword index pointer >> 2,
-; which discards the low two bits rather than accounting for them. An
-; under-aligned one would therefore reach the dword containing the address
-; instead of the bytes asked for - the same code as a correctly aligned access,
-; reading the wrong data with nothing to show for it. A sub-dword access
-; computes a bit offset instead, and is held only to its own natural alignment.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
-define i32 @load_i32_align1(ptr addrspace(13) inreg %p) {
- %x = load i32, ptr addrspace(13) %p, align 1
- ret i32 %x
-}
-
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only dword-aligned whole-dword and 8-/16-bit loads and stores are implemented
-define void @store_i32_align1(ptr addrspace(13) inreg %p, i32 %v) {
- store i32 %v, ptr addrspace(13) %p, align 1
- ret void
-}
-
-; Alignment is required of the pointer, not of the accessed type: a 64-bit
-; access needs only the dword alignment the index computation relies on.
-; CHECK-NOT: in function load_i64_align4
-define i64 @load_i64_align4(ptr addrspace(13) inreg %p) {
- %x = load i64, ptr addrspace(13) %p, align 4
- ret i64 %x
-}
-
-; And a byte access at align 1 is fine, which is what the whole-dword-only
-; restriction must not catch.
+; And an 8-bit access is held only to its own natural alignment, which is one
+; byte - the whole-dword rule must not catch it.
; CHECK-NOT: in function load_i8_align1
define i8 @load_i8_align1(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p, align 1
>From 8e204e13db5b6c14a13149b734fd8c3a990f4b77 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:12:39 +0300
Subject: [PATCH 03/21] Run AMDGPULowerIdxOps for optnone functions in
AMDGPULowerIdxOps
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 5 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 110 ++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 95cfe42df7e6d..e7d3a1e8bf172 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -168,8 +168,9 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: nothing else expands the
+ // sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
+ // therefore must not be skipped for optnone functions.
return LowerIdxOps(MF).run(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4b1703c1b6043..66790a4a155fe 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -490,3 +490,113 @@ define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
%y = zext i16 %x to i32
ret i32 %y
}
+
+; Expanding these pseudos is required lowering, so it has to happen for optnone
+; functions too - clang marks every function optnone at -O0. Nothing else can
+; lower them, so if the expansion is skipped AMDGPULowerVGPREncoding is left
+; with a pseudo it cannot handle.
+define i32 @load_i8_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-SDAG-LABEL: load_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_and_b32_e64 v0, v0, s0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnone {
+; GFX12-SDAG-LABEL: store_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ; kill: def $vgpr1_lo16 killed $vgpr0 killed $exec
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
>From b6810aa7155d3502ecc5e53193779cf5ef20a13a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:50:21 +0300
Subject: [PATCH 04/21] Add a MIR test for AMDGPULowerIdxOps
---
.../as-vgpr-lower-idx-ops.mir | 134 ++++++++++++++++++
1 file changed, 134 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
new file mode 100644
index 0000000000000..c991990db933e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+
+# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
+# AMDGPULowerIdxOps, in isolation:
+#
+# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+#
+# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
+# virtual here and the implicit-def $m0 that instruction selection attached is
+# carried over to the whole-dword accesses the expansion creates.
+
+---
+name: lower_load_idx_bits_zext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_zext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_sext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_sext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 1, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_16_offset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_16_offset
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 3, 16, 16, 0, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+# A dynamic bit offset is passed straight through to the bit-field extract;
+# only its low 5 bits are significant.
+---
+name: lower_load_idx_bits_dynamic_bitoffset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_dynamic_bitoffset
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, %1, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2
+...
+
+# The store is a read-modify-write of the containing dword. The index is read
+# again by the store, so it must not be killed by the load.
+---
+name: lower_store_idx_bits
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_store_idx_bits
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ V_STORE_IDX_BITS %1, killed %0, 0, 255, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ S_ENDPGM 0
+...
+
+---
+name: lower_store_idx_bits_dynamic_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: lower_store_idx_bits_dynamic_mask
+ ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = COPY $vgpr1
+ V_STORE_IDX_BITS %1, %0, 2, %2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ S_ENDPGM 0
+...
>From 07e31de69bb870452f3b39907e950488f3e91d9f Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:51:08 +0300
Subject: [PATCH 05/21] Verify operand invariants of the VGPR-memory indexed
accesses
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 28 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 26 +++++++++++++++++
2 files changed, 54 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 92426ae8e29b1..3d63444e8417a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5445,6 +5445,34 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
+ // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
+ // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
+ // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
+ // access has to record that the move clobbers M0, otherwise it could be
+ // separated from the code that sets M0 up.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (MI.getNumMemOperands() != 1) {
+ ErrInfo = "v_load/store_idx should have exactly one memory operand.";
+ return false;
+ }
+
+ const MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg()) {
+ if (IdxOp.getSubReg() != 0) {
+ ErrInfo =
+ "v_load/store_idx register index must not have a subregister.";
+ return false;
+ }
+
+ if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
+ !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
+ ErrInfo = "v_load/store_idx with register index must have implicit-def "
+ "$m0 on movrel subtargets.";
+ return false;
+ }
+ }
+ }
+
// Make sure the register classes are correct.
for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
new file mode 100644
index 0000000000000..bec5ef54aa203
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -0,0 +1,26 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
+# access ends up in M0, which the v_movrel[sd] emitted by
+# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
+# index to M0, the access must record that the eventual move clobbers M0 -
+# otherwise it could be separated from the code that sets M0 up. Check that the
+# machine verifier rejects a register index without that implicit-def, for both
+# the whole-dword and the sub-dword pseudos.
+
+# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
+
+---
+name: missing_m0_implicit_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
+ %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From b343d856843add11ef560be83eec62f5a29c7d6d Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:59:26 +0300
Subject: [PATCH 06/21] Test the bit-offset half of the VGPR-memory
demanded-bits combine
---
.../as-vgpr-index-demanded-bits.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index e1dfb50df7fb9..ef92bd4fbdfb1 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -7,6 +7,11 @@
; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
; path the mask must not survive into the M0 index computation.
+;
+; The same combine also trims the bit offset of a sub-dword access, which needs
+; only its low 5 bits to name a bit position within the containing dword. That
+; half matters independently: a mask can be dead for the bit offset while the
+; index computation keeps it alive, and vice versa.
define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-SDAG-LABEL: load_masked_index:
@@ -47,5 +52,79 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
store i32 %val, ptr addrspace(13) %ptr
ret void
}
+
+; A dynamic pointer masked with 0x3ff: the mask is dead both for the index and
+; for the bit offset, so no s_and may remain.
+define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %m = and i32 %sum, 1023
+ %ptr = inttoptr i32 %m to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
+
+; Setting a high bit of the pointer cannot affect the 5-bit bit offset either,
+; so no bit-set of the shifted offset may remain.
+define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_or_high_bit:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_or_high_bit:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %o = or i32 %sum, 65536
+ %ptr = inttoptr i32 %o to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
>From aa697b61ea99060245ceeaf59a3bfcd3666b1620 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:13:46 +0300
Subject: [PATCH 07/21] Simplify the sub-dword VGPR-memory pseudo definitions
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 57 +++++++++++-------------
1 file changed, 25 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 7a9790221f290..db9cfef134286 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1176,10 +1176,12 @@ defm : VRegIdxLoadStorePat<vt>;
//
// Only bitsize of 8 or 16 bits are supported and must be a compile-time
// constant.
-class V_LOAD_IDX_BITS<RegisterClass data_rc>
- : VPseudoInstSI<(outs data_rc:$data),
- (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
- VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+// The lowering normalizes the loaded value to a dword, so only a 32-bit $data
+// form is needed.
+def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
+ (ins SReg_32:$idx, i32imm:$offset,
+ i32imm:$bitsize, VSrc_b32:$bitoffset,
+ i32imm:$is_sext_load)> {
let mayLoad = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1187,15 +1189,13 @@ class V_LOAD_IDX_BITS<RegisterClass data_rc>
let Defs = [M0];
}
-def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
-
// Used for byte or short stores.
//
// $data holds the pre-shifted insert value and $mask holds the insert mask
// that is used when lowering to V_BFI_B32.
-class V_STORE_IDX_BITS<RegisterClass data_op_rc>
- : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
- i32imm:$offset, VSrc_b32:$mask)> {
+def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
+ (ins VGPR_32:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1203,32 +1203,25 @@ class V_STORE_IDX_BITS<RegisterClass data_op_rc>
let Defs = [M0];
}
-def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
-
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
// shape folds a constant dword offset into the pseudo's $offset operand.
-multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
- Instruction store_inst, ValueType vt> {
- def : GCNPat<
- (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
- (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
- imm:$is_sext_load)>;
- def : GCNPat<
- (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
- (store_inst $data, $idx, imm:$offset, $mask)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
- (store_inst $data, $idx, 0, $mask)>;
-}
-
-defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+def : GCNPat<
+ (i32 (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+def : GCNPat<
+ (i32 (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, imm:$offset, $mask)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, i32:$idx, i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, 0, $mask)>;
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
>From e80550627ed8bb9e86d0ceafa0a6c34425ed3357 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:24:43 +0300
Subject: [PATCH 08/21] Cover the remaining VGPR-memory operand invariants
---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 38 +++++++++++++------
1 file changed, 26 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
index bec5ef54aa203..db32b6957abb3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -1,26 +1,40 @@
# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
-# access ends up in M0, which the v_movrel[sd] emitted by
-# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
-# index to M0, the access must record that the eventual move clobbers M0 -
-# otherwise it could be separated from the code that sets M0 up. Check that the
-# machine verifier rejects a register index without that implicit-def, for both
-# the whole-dword and the sub-dword pseudos.
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
+# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
+# rewritten the index to M0, the access must record that the eventual move
+# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
+# The lowering also relies on the access having exactly one memory operand (it
+# synthesizes a load operand from it for a sub-dword store), and on the index
+# carrying no subregister, since it is rewritten in place to M0.
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
---
-name: missing_m0_implicit_def
+name: operand_invariants
tracksRegLiveness: true
body: |
bb.0:
- liveins: $sgpr0, $vgpr0
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
- %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
+
+ ; Missing implicit-def $m0, for each of the four pseudo forms.
+ %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %2, implicit %3
+ S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
...
>From 0ba03d3ac90f7f9ecec3c774ab67ff8eb7fe4c02 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:07:38 +0300
Subject: [PATCH 09/21] Also build the sub-dword VGPR-memory tests for wave64
---
llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 66790a4a155fe..992b737009947 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,6 +2,14 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
+; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
+; sub-dword code is the same there modulo register allocation, so only compile
+; and verify rather than checking the output a second time. What differs is the
+; waterfall around a divergent access, which uses a 64-bit exec mask.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
>From 3a5768c66e92462b42b5ed58ccc6a46b6e0f400d Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:28:47 +0300
Subject: [PATCH 10/21] Test sub-dword VGPR-memory accesses across control flow
---
.../as-vgpr-bits-control-flow.ll | 316 ++++++++++++++++++
1 file changed, 316 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
new file mode 100644
index 0000000000000..73edbbb7855e5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
+; across control flow. A sub-dword store is a read-modify-write of the dword
+; containing it, which makes it sensitive to how it is placed relative to other
+; accesses of the same dword, to loop bodies, and to the exec mask.
+
+; Two stores into different bytes of the same dword. Each is a full
+; read-modify-write, so the second has to observe the first: if the two dword
+; loads were merged, the first byte would be lost.
+define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
+; GFX12-SDAG-LABEL: two_bytes_same_dword:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: two_bytes_same_dword:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 1
+ store i8 %a, ptr addrspace(13) %p
+ store i8 %b, ptr addrspace(13) %q
+ ret void
+}
+
+; A store inside a loop. The pointer deliberately depends on the induction
+; variable: with a loop-invariant pointer the whole access is hoisted out and
+; nothing inside the loop is exercised. M0 has to be re-established for each
+; access on every iteration.
+define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
+; GFX12-SDAG-LABEL: store_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ store i8 %v, ptr addrspace(13) %q
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Likewise for a load, accumulating so the result cannot be folded away.
+define i32 @load_in_loop(ptr addrspace(13) inreg %p, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: .LBB2_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: .LBB2_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ %x = load i8, ptr addrspace(13) %q
+ %z = zext i8 %x to i32
+ %acc.next = add i32 %acc, %z
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; A store under divergent control flow: the read-modify-write must stay inside
+; the conditional region so that lanes which did not take the branch keep their
+; dword unmodified.
+define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
+; GFX12-SDAG-LABEL: store_divergent_if:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
+; GFX12-SDAG-NEXT: ; %bb.1: ; %then
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_divergent_if:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-GISEL-NEXT: ; %bb.1: ; %then
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %c = icmp ult i32 %tid, 8
+ br i1 %c, label %then, label %exit
+
+then:
+ store i8 %v, ptr addrspace(13) %p
+ br label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From b621431e0eab77ec7fb7b1565368017a30a0324c Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:59:30 +0300
Subject: [PATCH 11/21] NFC: track the downstream implementation more closely
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 35 ++++++++++++--------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
2 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index e7d3a1e8bf172..815605099cd90 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -36,9 +36,9 @@ using namespace llvm;
namespace {
-class LowerIdxOps {
+class AMDGPULowerIdxOpsImpl {
public:
- LowerIdxOps(MachineFunction &MF)
+ AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
MRI(&MF.getRegInfo()) {}
@@ -52,7 +52,7 @@ class LowerIdxOps {
MachineRegisterInfo *MRI;
};
-void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -69,19 +69,21 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
- LoadMIB.addMemOperand(*MI.memoperands_begin());
+ auto *LoadMMO = *MI.memoperands_begin();
+ LoadMIB.addMemOperand(LoadMMO);
// Match what instruction selection does for a whole-dword access with a
// register index: record that the M0 write implied by the eventual movrel
// clobbers M0 (see AMDGPUAssignIdxToM0).
if (LoadIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+ Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
MachineOperand BitOffset = MI.getOperand(4);
if (BitOffset.isImm())
BitOffset.setImm(BitOffset.getImm() & 31);
- BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, DataReg)
.addReg(SrcAReg)
.add(BitOffset)
.add(MI.getOperand(3)); // bitsize
@@ -91,11 +93,15 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent();
auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+ const MCInstrDesc &II = TII->get(AMDGPU::V_BFI_B32_e64);
+
+ Register DataReg = StoreIdx.getDataOp().getReg();
+
Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
@@ -110,15 +116,16 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
// Synthesize a load MMO from the store's.
auto NewFlags = MachineMemOperand::MOLoad;
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
- LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
+ LoadMIB.addMemOperand(LoadMMO);
if (StoreIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
- BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
- .add(StoreIdx.getOperand(3)) // mask
- .addReg(StoreIdx.getDataOp().getReg())
- .addReg(SrcAReg);
+ auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
+ CoreMIB.add(StoreIdx.getOperand(3)); // mask
+ CoreMIB.addReg(DataReg);
+ CoreMIB.addReg(SrcAReg);
// Write the dword back.
auto StoreMIB =
@@ -135,7 +142,7 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-bool LowerIdxOps::run(MachineFunction &MF) {
+bool AMDGPULowerIdxOpsImpl::run(MachineFunction &MF) {
bool Changed = false;
LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
@@ -171,7 +178,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
// This is required lowering, not an optimization: nothing else expands the
// sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
// therefore must not be skipped for optnone functions.
- return LowerIdxOps(MF).run(MF);
+ return AMDGPULowerIdxOpsImpl(MF).run(MF);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -187,7 +194,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
PreservedAnalyses
AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!LowerIdxOps(MF).run(MF))
+ if (!AMDGPULowerIdxOpsImpl(MF).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 3c12064a088a2..5d817d0b7c588 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19509,7 +19509,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
+ unsigned IndexOpIdx, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
>From 6e4289543d59f9bb16b56ae7a03f7a378ed33079 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:16:13 +0300
Subject: [PATCH 12/21] Do not describe sub-dword VGPR-memory accesses by index
and width
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3d63444e8417a..af72f2255ee00 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -664,6 +664,13 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
}
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ // A sub-dword access is not described by a dword index and width alone -
+ // the bit position within the dword is part of the address - and has no
+ // entry in the width table. Report it as opaque so that callers such as
+ // areMemAccessesTriviallyDisjoint() fall back to assuming an overlap.
+ if (!AMDGPU::getVLdStIdxOpcodeInfoByOpcode(LdSt.getOpcode()))
+ return false;
+
BaseOp = &LdStIdx->getIdxOp();
OffsetOp = &LdStIdx->getOffsetOp();
>From 2a1620deb87fbb3ab5b3b43730f2868f83694f4e Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:47:57 +0300
Subject: [PATCH 13/21] Fix tests
---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 15 ++-
.../as-vgpr-index-demanded-bits.ll | 105 ++++++++++++++++++
2 files changed, 115 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 992b737009947..4910e9c56c4b7 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,13 +2,18 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
-; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
-; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
-; sub-dword code is the same there modulo register allocation, so only compile
-; and verify rather than checking the output a second time. What differs is the
-; waterfall around a divergent access, which uses a 64-bit exec mask.
+; Also compile for two further movrel subtargets. The expansion itself is
+; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
+; the sub-dword code is the same on both modulo register allocation, so only
+; compile and verify rather than checking the output again. gfx942 differs in
+; using a 64-bit exec mask for the waterfall around a divergent access, and
+; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
+; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
+; functions also cover the S_SET_VGPR_MSB encoding.
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index ef92bd4fbdfb1..4948db66c3023 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
@@ -29,6 +31,28 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
%v = load i32, ptr addrspace(13) %ptr
@@ -47,6 +71,24 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-LABEL: store_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: store_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_endpgm
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
store i32 %val, ptr addrspace(13) %ptr
@@ -82,6 +124,37 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%m = and i32 %sum, 1023
%ptr = inttoptr i32 %m to ptr addrspace(13)
@@ -119,6 +192,37 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%o = or i32 %sum, 65536
%ptr = inttoptr i32 %o to ptr addrspace(13)
@@ -128,3 +232,4 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
+; GFX1250: {{.*}}
>From f2fb95bdfad5d1bbafafa52e5a7fe4b19367056a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 18:14:45 +0300
Subject: [PATCH 14/21] Check the encodings of the sub-dword VGPR-memory
accesses
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 12 +---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 26 +++----
.../as-vgpr-bits-control-flow.ll | 7 +-
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 71 +++++++++++++++++++
.../as-vgpr-lower-idx-ops.mir | 16 ++---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 40 -----------
.../AddressSpaceVGPR/as-vgpr-verifier.mir | 32 +++++++++
7 files changed, 125 insertions(+), 79 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 815605099cd90..ec1463e0b2eb2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -15,7 +15,8 @@
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup, and before register allocation
+/// here take part in the usual M0 setup - they declare their write of M0
+/// themselves, like the pseudo they replace - and before register allocation
/// because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -71,11 +72,6 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
.add(LoadIdx.getOffsetOp());
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
- // Match what instruction selection does for a whole-dword access with a
- // register index: record that the M0 write implied by the eventual movrel
- // clobbers M0 (see AMDGPUAssignIdxToM0).
- if (LoadIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
@@ -118,8 +114,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
LoadMIB.addMemOperand(LoadMMO);
- if (StoreIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
@@ -134,8 +128,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
StoreMIB.addMemOperand(StoreMMO);
- if (StoreIdx.getIdxOp().isReg())
- StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index af72f2255ee00..f569c85ba3576 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5452,11 +5452,11 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
- // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
- // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
- // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
- // access has to record that the move clobbers M0, otherwise it could be
- // separated from the code that sets M0 up.
+ // The lowering of a VGPR "as memory" indexed access relies on it having
+ // exactly one memory operand, since a sub-dword store synthesizes a load
+ // operand from it, and on the index carrying no subregister, since
+ // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
+ // is declared by the instruction itself, so it needs no check here.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
@@ -5464,19 +5464,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
const MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg()) {
- if (IdxOp.getSubReg() != 0) {
- ErrInfo =
- "v_load/store_idx register index must not have a subregister.";
- return false;
- }
-
- if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
- !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
- ErrInfo = "v_load/store_idx with register index must have implicit-def "
- "$m0 on movrel subtargets.";
- return false;
- }
+ if (IdxOp.isReg() && IdxOp.getSubReg() != 0) {
+ ErrInfo = "v_load/store_idx register index must not have a subregister.";
+ return false;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 73edbbb7855e5..1f20bf28b2335 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -37,15 +37,16 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
new file mode 100644
index 0000000000000..32921a6244ebe
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -0,0 +1,71 @@
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+
+; The sub-dword accesses of the VGPR "as memory" address space (13) that
+; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
+; check assembly text, which does not show that the operand forms the expansion
+; picks can be assembled and encoded: a mask it builds from a known bit offset is
+; too wide for an inline constant and becomes a 32-bit literal, and a bit offset
+; reaches the bit-field extract as either an inline constant or an SGPR. On a
+; subtarget with more than 256 addressable VGPRs the whole-dword accesses the
+; expansion creates additionally carry the S_SET_VGPR_MSB encoding, so check
+; there too.
+;
+; The expansion runs after instruction selection and is handed the same pseudos
+; by both selectors, which produce identical code for these functions, so one
+; selector is enough here. as-vgpr-bits.ll covers the two separately.
+
+; A statically known bit offset turns the mask into a literal operand of the
+; bit-field insert.
+define amdgpu_ps void @store_i8_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_aligned>:
+; GFX1250: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i16_aligned>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xffff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i16
+ store i16 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; Without one the mask is computed at run time and is an SGPR instead.
+define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_dynamic_mask>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; Both bit offset and width of the extract are inline constants here.
+define amdgpu_ps i32 @load_i8_zext_aligned(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_zext_aligned>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_u32 v0, v0, 0, 8
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %z = zext i8 %x to i32
+ ret i32 %z
+}
+
+; A run-time bit offset is an SGPR, and the sign-extending load selects the
+; signed extract.
+define amdgpu_ps i32 @load_i8_sext_dynamic(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_sext_dynamic>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_i32 v0, v0, s0, 8
+ %x = load i8, ptr addrspace(13) %p
+ %s = sext i8 %x to i32
+ ret i32 %s
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c991990db933e..c8b4fefa3c3e3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -21,7 +21,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +39,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +57,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +78,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +100,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +122,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
deleted file mode 100644
index db32b6957abb3..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ /dev/null
@@ -1,40 +0,0 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-
-# Operand invariants of the VGPR "as memory" (address space 13) indexed
-# accesses, as enforced by SIInstrInfo::verifyInstruction.
-#
-# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
-# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
-# rewritten the index to M0, the access must record that the eventual move
-# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
-# The lowering also relies on the access having exactly one memory operand (it
-# synthesizes a load operand from it for a sub-dword store), and on the index
-# carrying no subregister, since it is rewritten in place to M0.
-
-# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
-# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
-# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
-
----
-name: operand_invariants
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
- %0:sreg_32 = COPY $sgpr0
- %1:vgpr_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr2_sgpr3
-
- ; No memory operand.
- %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
-
- ; Index with a subregister.
- %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
-
- ; Missing implicit-def $m0, for each of the four pseudo forms.
- %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
- V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
- V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
-...
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
new file mode 100644
index 0000000000000..d4e7ec49e1f40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -0,0 +1,32 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# The lowering relies on the access having exactly one memory operand, since a
+# sub-dword store synthesizes a load operand from it, and on the index carrying
+# no subregister, since AMDGPUAssignIdxToM0 rewrites it in place to M0. That the
+# access writes M0 is declared by the instructions themselves, so it needs no
+# check: MIR without the operand does not parse.
+
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
+
+---
+name: operand_invariants
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit-def $m0, implicit $exec
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit-def $m0, implicit $exec :: (load (s32), addrspace 13)
+
+ S_ENDPGM 0, implicit %3, implicit %4
+...
>From 9b3c2fce1cbfe4889a7a9fed5cfc6e4ef3ee177a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 31 Jul 2026 20:15:02 +0300
Subject: [PATCH 15/21] Document the sub-dword access rules for the VGPR
address space
---
llvm/docs/AMDGPUUsage.rst | 7 +++++
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 26 ++++++++++---------
2 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index cb4d447d3766f..4840f2e3b6648 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1376,6 +1376,13 @@ supported for the ``amdgcn`` target.
of ``v3``. Storing 8 bytes to address ``32`` overwrites the contents of
``v[8:9]``.
+ An access narrower than a dword is implemented by extracting or inserting the
+ bits within the dword that contains it, so it must not straddle a dword
+ boundary: 8-bit accesses are unrestricted, while 16-bit accesses must be
+ 2-byte aligned. Wider accesses must be a whole number of dwords. Anything
+ else, such as an under-aligned 16-bit access, is reported as an error rather
+ than miscompiled.
+
Use of this address space by frontends is strongly discouraged. It has unusual
and subtle lifetime rules due to the potential for interaction with normal
register allocation. It exists primarily for internal purposes of the backend,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 62c3264f9f7ec..1551d46aed7fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1845,7 +1845,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
// unsupported access is diagnosed cleanly rather than failing to
// legalize.
- .customIf([=](const LegalityQuery &Query) -> bool {
+ .customIf([](const LegalityQuery &Query) -> bool {
return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
@@ -3583,7 +3583,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register BaseReg = PtrReg;
int64_t Offset = 0;
if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
- if (auto MaybeOff =
+ if (std::optional<ValueAndVReg> MaybeOff =
getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
BaseReg = PtrAdd->getBaseReg();
Offset = MaybeOff->Value.getSExtValue();
@@ -3596,7 +3596,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
HaveConstantBitOffset = true;
ConstantBitOffsetVal = 0;
} else {
- auto &VT = *Helper.getValueTracking();
+ GISelValueTracking &VT = *Helper.getValueTracking();
KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
if (BaseKB.isConstant()) {
Offset += BaseKB.getConstant().getZExtValue();
@@ -3606,18 +3606,18 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
}
// Setup common registers.
- const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
- auto Two = B.buildConstant(I32, 2);
- const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+ const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ MachineInstrBuilder Two = B.buildConstant(I32, 2);
+ const MachineInstrBuilder Index = B.buildLShr(I32, PtrAsInt, Two);
- const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ const MachineInstrBuilder BitWidthReg = B.buildConstant(I32, MemSize);
Register BitOffsetReg;
if (HaveConstantBitOffset) {
BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
} else {
// V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
// offset, so masking the high bits is unnecessary.
- auto Three = B.buildConstant(I32, 3);
+ MachineInstrBuilder Three = B.buildConstant(I32, 3);
BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
}
@@ -3631,19 +3631,21 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
Register ShiftAmt = BitOffsetReg;
if (!HaveConstantBitOffset) {
- auto Mask = B.buildConstant(I32, 31);
+ MachineInstrBuilder Mask = B.buildConstant(I32, 31);
ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
}
Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
- const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
- const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+ const MachineInstrBuilder MaskBase =
+ B.buildConstant(I32, (1u << MemSize) - 1);
+ const MachineInstrBuilder Mask = B.buildShl(I32, MaskBase, ShiftAmt);
B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
.addMemOperand(&MMO);
} else {
const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
- const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ const MachineInstrBuilder IsSExtReg =
+ B.buildConstant(I32, IsSExt ? 1 : 0);
Register Result =
B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
{Index, BitWidthReg, BitOffsetReg, IsSExtReg})
>From 624daddf32b12c6ba083b3441527684ec6683371 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 3 Aug 2026 20:04:45 +0300
Subject: [PATCH 16/21] Adopt subarch triples in the sub-dword tests and drop
the redundant VALU
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../AddressSpaceVGPR/as-vgpr-bits-control-flow.ll | 8 ++++----
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 4 ++--
.../CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 12 ++++++------
.../AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir | 2 +-
6 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index db9cfef134286..d16f2dc0cc669 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1183,7 +1183,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
i32imm:$bitsize, VSrc_b32:$bitoffset,
i32imm:$is_sext_load)> {
let mayLoad = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
@@ -1197,7 +1196,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
(ins VGPR_32:$data, SReg_32:$idx,
i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
- let VALU = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
let Defs = [M0];
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 1f20bf28b2335..23bb00886d179 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
; across control flow. A sub-dword store is a read-modify-write of the dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 32921a6244ebe..504e8034658cf 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
-; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.00-- -o - %s | llvm-mc -triple=amdgpu12.00-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -mtriple=amdgpu12.50-- -o - %s | llvm-mc -triple=amdgpu12.50-- -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
; The sub-dword accesses of the VGPR "as memory" address space (13) that
; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4910e9c56c4b7..9e7bea5d5f8cd 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
; Also compile for two further movrel subtargets. The expansion itself is
; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
@@ -10,10 +10,10 @@
; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
; functions also cover the S_SET_VGPR_MSB encoding.
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
-; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
-; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42-- -filetype=null %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -filetype=null %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-- -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c8b4fefa3c3e3..03646b899cbae 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu12.00-- -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
# AMDGPULowerIdxOps, in isolation:
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
index d4e7ec49e1f40..f42d75a904bfc 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgpu12.00-- -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
# Operand invariants of the VGPR "as memory" (address space 13) indexed
# accesses, as enforced by SIInstrInfo::verifyInstruction.
>From 8c6abb834d4419f20164074746de9c64cf45c739 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 15:22:07 +0300
Subject: [PATCH 17/21] Regenerate the sub-dword expectations for the M0 setup
change
---
.../as-vgpr-bits-control-flow.ll | 88 ++++++++-----------
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 2 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 79 +++++++----------
3 files changed, 67 insertions(+), 102 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 23bb00886d179..51f5e1fb468ff 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -22,31 +22,27 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -58,32 +54,29 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -107,10 +100,9 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
@@ -119,7 +111,6 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
@@ -136,20 +127,18 @@ define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, s3, v0
; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v2, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
@@ -257,16 +246,14 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
; GFX12-SDAG-NEXT: ; %bb.1: ; %then
; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
@@ -285,19 +272,16 @@ define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
; GFX12-GISEL-NEXT: ; %bb.1: ; %then
; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
index 504e8034658cf..5638ce53c394b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -42,7 +42,7 @@ define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %
define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
; DIS-LABEL: <store_i8_dynamic_mask>:
; DIS: v_movrels_b32_e32 v1, v0
-; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_bfi_b32 v0, s0, v0, v1
; DIS: v_movreld_b32_e32 v0, v0
%v = trunc i32 %vv to i8
store i8 %v, ptr addrspace(13) %p
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 9e7bea5d5f8cd..1c3f1db234aec 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -207,16 +207,14 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -229,19 +227,16 @@ define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
@@ -257,16 +252,14 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -279,19 +272,16 @@ define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
-; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, 0xffff, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i16 %v, ptr addrspace(13) %p
@@ -306,11 +296,8 @@ define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s0, s0, 2
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
; GFX12-NEXT: v_movrels_b32_e32 v1, v0
-; GFX12-NEXT: s_mov_b32 m0, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
; GFX12-NEXT: v_movreld_b32_e32 v0, v0
@@ -403,11 +390,10 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
@@ -444,7 +430,6 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
@@ -568,15 +553,13 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, s1
; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
-; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -589,25 +572,23 @@ define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnon
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 31
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_and_b32 s1, s0, s1
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 0xff
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s1, s0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
store i8 %v, ptr addrspace(13) %p
>From 6871e98b58c6581891ba43c476c90cdf2f1e3e71 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 4 Aug 2026 17:43:50 +0300
Subject: [PATCH 18/21] Inherit the M0 operand when expanding the sub-dword
VGPR-memory pseudos
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 25 ++++++++++++++++---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +--
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 --
.../as-vgpr-lower-idx-ops.mir | 23 +++++++++--------
4 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index ec1463e0b2eb2..ba632ce02074c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -14,10 +14,9 @@
/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
-/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup - they declare their write of M0
-/// themselves, like the pseudo they replace - and before register allocation
-/// because it introduces new virtual registers.
+/// This runs after SITargetLowering::finalizeLowering has given the pseudo its
+/// M0 operand, so the whole-dword accesses created here inherit it, and before
+/// register allocation because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -41,6 +40,7 @@ class AMDGPULowerIdxOpsImpl {
public:
AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ TRI(MF.getSubtarget<GCNSubtarget>().getRegisterInfo()),
MRI(&MF.getRegInfo()) {}
bool run(MachineFunction &MF);
@@ -48,11 +48,25 @@ class AMDGPULowerIdxOpsImpl {
private:
void lowerLoadIdxBits(MachineInstr &MI);
void lowerStoreIdxBits(MachineInstr &MI);
+ void inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const;
const SIInstrInfo *TII;
+ const SIRegisterInfo *TRI;
MachineRegisterInfo *MRI;
};
+// An indexed access takes its index from M0 where the subtarget has movrel, and
+// clobbers M0 through s_set_gpr_idx_on where it indexes with the VGPR indexing
+// mode. SITargetLowering::finalizeLowering picks between the two and records a
+// clobber as an implicit def, so a whole-dword access replacing the pseudo here
+// has to carry that over: the index itself comes along with the operand.
+void AMDGPULowerIdxOpsImpl::inheritM0Def(const MachineInstr &MI,
+ const MachineInstrBuilder &MIB) const {
+ if (MI.definesRegister(AMDGPU::M0, TRI))
+ MIB.addDef(AMDGPU::M0, RegState::Implicit);
+}
+
void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -70,6 +84,7 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
Register DataReg = LoadIdx.getDataOp().getReg();
@@ -106,6 +121,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, LoadMIB);
// The index is read again by the store below, so it does not die here.
LoadMIB->getOperand(1).setIsKill(false);
auto *StoreMMO = *MI.memoperands_begin();
@@ -127,6 +143,7 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.addReg(DstAReg)
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
+ inheritM0Def(MI, StoreMIB);
StoreMIB.addMemOperand(StoreMMO);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f569c85ba3576..ae76fdbde74c1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5455,8 +5455,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
// The lowering of a VGPR "as memory" indexed access relies on it having
// exactly one memory operand, since a sub-dword store synthesizes a load
// operand from it, and on the index carrying no subregister, since
- // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0
- // is declared by the instruction itself, so it needs no check here.
+ // SITargetLowering::finalizeLowering rewrites it in place to M0.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d16f2dc0cc669..a420958595373 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1185,7 +1185,6 @@ def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
let mayLoad = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Used for byte or short stores.
@@ -1198,7 +1197,6 @@ def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
let mayStore = 1;
let UseNamedOperandTable = 1;
let hasSideEffects = 0;
- let Defs = [M0];
}
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index 03646b899cbae..c5270bd219141 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -7,9 +7,10 @@
# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
#
-# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
-# virtual here and the implicit-def $m0 that instruction selection attached is
-# carried over to the whole-dword accesses the expansion creates.
+# The index and any M0 clobber are given to the pseudo beforehand, by
+# SITargetLowering::finalizeLowering, so the whole-dword accesses the expansion
+# creates carry both over: the index as their own index operand, and the clobber
+# as an implicit def where the subtarget indexes with the VGPR indexing mode.
---
name: lower_load_idx_bits_zext
@@ -21,7 +22,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +40,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +58,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +79,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +101,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +123,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
>From fe6a59b0365905a545e7836dc546dc8215ea56ea Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Fri, 14 Aug 2026 17:00:57 +0200
Subject: [PATCH 19/21] Regenerate the demanded-bits expectations for the
gfx1250 entry sequence
---
.../as-vgpr-index-demanded-bits.ll | 40 +++++++++++--------
1 file changed, 24 insertions(+), 16 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 4948db66c3023..2bf0ba0d09281 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -34,9 +34,10 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
;
; GFX1250-SDAG-LABEL: load_masked_index:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -45,9 +46,10 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
;
; GFX1250-GISEL-LABEL: load_masked_index:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -74,18 +76,20 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
;
; GFX1250-SDAG-LABEL: store_masked_index:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: store_masked_index:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX1250-GISEL-NEXT: s_endpgm
@@ -127,9 +131,10 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
;
; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
@@ -142,9 +147,10 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
;
; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
@@ -195,9 +201,10 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
;
; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: global_wb
-; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
@@ -210,9 +217,10 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
;
; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: global_wb
-; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
>From 929d2a76adb0092c631189a70a41ad01659180e4 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 13:29:37 -0400
Subject: [PATCH 20/21] Treat a sub-dword VGPR-memory load as divergent too
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 83 +++++++++++++++++++
2 files changed, 86 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5d817d0b7c588..1e9d002c0307d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20901,11 +20901,13 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS ||
AS == AMDGPUAS::VGPR;
}
- // The lowered form of the above. Without this the DAG takes the node's
+ // The lowered forms of the above: a whole-dword access becomes REG_LOAD, a
+ // sub-dword one REG_LOAD_BITS. Without these the DAG takes a node's
// divergence to be that of its operands, so a uniform index makes the loaded
// value look uniform, and a consumer that requires a uniform operand gets a
// v_readfirstlane - broadcasting one lane's value to the whole wave.
case AMDGPUISD::REG_LOAD:
+ case AMDGPUISD::REG_LOAD_BITS:
return true;
case ISD::CALLSEQ_END:
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index e779e6ff4d334..eca66ddc05672 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -295,6 +295,89 @@ define i32 @uniform_index_divergent_value(ptr addrspace(13) inreg %p, <4 x i32>
ret i32 %r
}
+; The same holds for a sub-dword access, which lowers to a different node. Its
+; value is per-lane too, so a uniform index does not make it uniform.
+define i32 @uniform_index_divergent_byte(ptr addrspace(13) inreg %p, <4 x i32> inreg %rsrc) {
+; GFX12-SDAG-LABEL: uniform_index_divergent_byte:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX12-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX12-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: uniform_index_divergent_byte:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[4:7], null offen
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: uniform_index_divergent_byte:
+; GFX942-SDAG: ; %bb.0:
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s4, s1
+; GFX942-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX942-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX942-SDAG-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX942-SDAG-NEXT: s_set_gpr_idx_off
+; GFX942-SDAG-NEXT: s_mov_b32 s7, s16
+; GFX942-SDAG-NEXT: s_mov_b32 s6, s3
+; GFX942-SDAG-NEXT: s_mov_b32 s5, s2
+; GFX942-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX942-SDAG-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: uniform_index_divergent_byte:
+; GFX942-GISEL: ; %bb.0:
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_mov_b32 s4, s1
+; GFX942-GISEL-NEXT: s_lshr_b32 s1, s0, 2
+; GFX942-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_on s1, gpr_idx(SRC0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-GISEL-NEXT: s_set_gpr_idx_off
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s2
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s16
+; GFX942-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX942-GISEL-NEXT: buffer_load_dword v0, v0, s[4:7], 0 offen
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %v = load i8, ptr addrspace(13) %p, align 1
+ %z = zext i8 %v to i32
+ %r = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %rsrc, i32 %z, i32 0)
+ ret i32 %r
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
; GFX942: {{.*}}
>From c5170f3bd29519ae1a21d8881d6e1c7a24ab5096 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 26 Aug 2026 15:30:55 -0400
Subject: [PATCH 21/21] Pin the width guard that keeps sub-dword accesses out
of aliasing decisions
---
llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp | 66 +++++++++++++++++++
1 file changed, 66 insertions(+)
diff --git a/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
index 01e628a67e9f2..c36970a783e4d 100644
--- a/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
+++ b/llvm/unittests/Target/AMDGPU/VGPRAsMemory.cpp
@@ -123,3 +123,69 @@ body: |
}
}
}
+
+// A sub-dword access is not described by a dword index and width alone - the
+// bit position within the dword is part of the address - so these pseudos carry
+// no entry in the width table, deliberately. getMemOperandsWithOffsetWidth must
+// decline to describe one rather than reaching getBitWidth() and the
+// llvm_unreachable behind it. With assertions off that unreachable is UB, and
+// the caller it would reach through is an aliasing decision, so the failure
+// would be a garbage width silently declaring two overlapping accesses
+// disjoint.
+//
+// Note the class does match these pseudos, so the dyn_cast is not what protects
+// the call; the width-table lookup inside it is.
+TEST_F(VGPRAsMemoryTest, SubDwordAccessHasNoDescribableWidth) {
+ StringRef MIRString = R"MIR(
+name: width
+body: |
+ bb.0:
+ liveins: $m0, $sgpr0, $vgpr0
+
+ $vgpr1 = V_LOAD_IDX_B32 $m0, 0, implicit $exec :: (load (s32), addrspace 13)
+ $vgpr2 = V_LOAD_IDX_BITS $m0, 0, 8, $sgpr0, 0, implicit $exec :: (load (s8), addrspace 13)
+ $vgpr3 = V_LOAD_IDX_BITS $m0, 0, 8, $sgpr0, 0, implicit $exec :: (load (s8), addrspace 13)
+ S_ENDPGM 0
+...
+)MIR";
+
+ ASSERT_TRUE(parseMIR(MIRString));
+ MachineFunction &MF = getMF("width");
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ const TargetRegisterInfo *TRI = ST.getRegisterInfo();
+ MachineBasicBlock *MBB = MF.getBlockNumbered(0);
+
+ auto Describe = [&](const MachineInstr &MI) {
+ SmallVector<const MachineOperand *, 4> BaseOps;
+ int64_t Offset = 0;
+ bool OffsetIsScalable = false;
+ LocationSize Width = LocationSize::precise(0);
+ return TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
+ OffsetIsScalable, Width, TRI);
+ };
+
+ const MachineInstr *FirstBits = nullptr;
+ const MachineInstr *SecondBits = nullptr;
+ for (MachineInstr &MI : *MBB) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_B32:
+ // A whole-dword access is describable: dword index plus a known width.
+ EXPECT_TRUE(Describe(MI));
+ break;
+ case AMDGPU::V_LOAD_IDX_BITS:
+ EXPECT_FALSE(Describe(MI))
+ << "a sub-dword access must be reported as opaque";
+ (FirstBits ? SecondBits : FirstBits) = &MI;
+ break;
+ default:
+ break;
+ }
+ }
+
+ // And the caller that would otherwise reach the unreachable: with no width to
+ // compare, two sub-dword accesses have to be assumed to overlap.
+ ASSERT_NE(FirstBits, nullptr);
+ ASSERT_NE(SecondBits, nullptr);
+ EXPECT_FALSE(TII->areMemAccessesTriviallyDisjoint(*FirstBits, *SecondBits));
+}
More information about the llvm-branch-commits
mailing list