[llvm-branch-commits] [llvm] [AMDGPU] Add support for sub-dword types for address space 13 (PR #213223)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 31 01:30:32 PDT 2026
https://github.com/doru1004 created https://github.com/llvm/llvm-project/pull/213223
Adds 8- and 16-bit loads and stores for the VGPR "as memory" address space (13).
A sub-dword access happens inside the dword that contains it: a load reads that
dword with the existing indexed access and extracts the bits with
`v_bfe_{u,i}32`, while a store is a read-modify-write that inserts them with
`v_bfi_b32`. A new pass, `AMDGPULowerIdxOps`, expands the sub-dword pseudos into
that sequence before register allocation.
Since the bit-field operations work within one register, a sub-dword access must
not straddle a dword boundary: 8-bit accesses are unrestricted, 16-bit accesses
must be 2-byte aligned, and anything else is diagnosed rather than silently
miscompiled.
Depends on #209541.
Co-authored-by: Nicolai Hähnle <Nicolai.Haehnle at amd.com>
>From 3a6d0e7f6e287dbc5b833b61f766b95aca7fd6c3 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 12:39:47 +0300
Subject: [PATCH 01/15] Add support for sub-dword types for address space 13
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 9 +
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 2 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 146 +++++-
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 201 ++++++++
.../lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp | 9 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 6 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 18 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 10 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 138 +++++-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 18 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 82 ++++
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 449 ++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 41 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +
19 files changed, 1106 insertions(+), 51 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index ad9cc6aa12c1d..0f24f624b3f8e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeAMDGPUAssignIdxToM0LegacyPass(PassRegistry &);
extern char &AMDGPUAssignIdxToM0ID;
+void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
+extern char &AMDGPULowerIdxOpsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -446,6 +449,12 @@ class AMDGPUAssignIdxToM0Pass : public PassInfoMixin<AMDGPUAssignIdxToM0Pass> {
MachineFunctionAnalysisManager &MFAM);
};
+class AMDGPULowerIdxOpsPass : public PassInfoMixin<AMDGPULowerIdxOpsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 38a692a45fa29..7143bb682a414 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -284,6 +284,8 @@ def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
def : GINodeEquiv<G_AMDGPU_REG_LOAD, SIreg_load>;
def : GINodeEquiv<G_AMDGPU_REG_STORE, SIreg_store>;
+def : GINodeEquiv<G_AMDGPU_REG_LOAD_BITS, SIreg_load_bits>;
+def : GINodeEquiv<G_AMDGPU_REG_STORE_BITS, SIreg_store_bits>;
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2e666845e5637..230cdd2a19c02 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -556,12 +556,20 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Only whole-dword
-// accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported
-// for now; sub-dword (8/16-bit) support lands later.
+// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
+// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
+// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
- return MemSize == ValSize &&
- AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+ if (MemSize == 8 || MemSize == 16) {
+ if (ValSize == MemSize)
+ return true;
+ if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
+ return true;
+ return false;
+ }
+ if (MemSize != ValSize)
+ return false;
+ return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
}
/// Return true if a load or store of the type should be lowered with a bitcast
@@ -1813,6 +1821,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
{S16, ConstantPtr, S8, GlobalAlign8}})
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
+ })
+ // VGPR ("as memory") extending loads are custom-lowered to
+ // G_AMDGPU_REG_LOAD_BITS. Always take the custom path so an
+ // unsupported access is diagnosed cleanly rather than failing to
+ // legalize.
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ return Query.Types[1].getAddressSpace() == AMDGPUAS::VGPR;
});
if (ST.hasFlatAddressSpace()) {
@@ -3473,18 +3488,21 @@ static LLT widenToNextPowerOf2(LLT Ty) {
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
-/// Lower a whole-dword G_LOAD / G_STORE on AMDGPUAS::VGPR into a legal
-/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE indexed by the pointer's dword offset
-/// (pointer >> 2). Parallels the SelectionDAG LowerLoadStoreVGPR.
+/// Lower a G_LOAD / G_STORE / G_SEXTLOAD / G_ZEXTLOAD on AMDGPUAS::VGPR into a
+/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
+/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
+/// Parallels the SelectionDAG LowerLoadStoreVGPR.
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
MachineMemOperand &MMO = **MI.memoperands_begin();
- const bool IsStore = MI.getOpcode() == AMDGPU::G_STORE;
+ const unsigned Opcode = MI.getOpcode();
+ const bool IsStore = Opcode == AMDGPU::G_STORE;
Register ValReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
+ const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
const LLT ValTy = MRI.getType(ValReg);
const unsigned ValSize = ValTy.getSizeInBits();
// The GISel selection patterns for the indexed pseudos - and for the shift /
@@ -3493,16 +3511,14 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic instead of failing to legalize
- // (sub-dword support lands in a later change).
- if (!isVGPRLoadStoreSizeSupported(MMO.getMemoryType().getSizeInBits(),
- ValSize)) {
+ // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
+ // with a diagnostic instead of failing to legalize.
+ if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize)) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "whole-dword and 8-/16-bit loads and stores are implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
@@ -3510,6 +3526,102 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
}
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (MemSize < 32) {
+ assert(MemSize == 8 || MemSize == 16);
+ assert(MemSize <= ValSize && ValSize <= 32);
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // known constant.
+ Register BaseReg = PtrReg;
+ int64_t Offset = 0;
+ if (auto *PtrAdd = getOpcodeDef<GPtrAdd>(PtrReg, MRI)) {
+ if (auto MaybeOff =
+ getIConstantVRegValWithLookThrough(PtrAdd->getOffsetReg(), MRI)) {
+ BaseReg = PtrAdd->getBaseReg();
+ Offset = MaybeOff->Value.getSExtValue();
+ }
+ }
+
+ bool HaveConstantBitOffset = false;
+ int64_t ConstantBitOffsetVal = 0;
+ if (Offset == 0 && MMO.getAlign() >= Align(4)) {
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = 0;
+ } else {
+ auto &VT = *Helper.getValueTracking();
+ KnownBits BaseKB = VT.getKnownBits(BaseReg).trunc(2);
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ HaveConstantBitOffset = true;
+ ConstantBitOffsetVal = (Offset & 3) * 8;
+ }
+ }
+
+ // Setup common registers.
+ const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
+ auto Two = B.buildConstant(I32, 2);
+ const auto Index = B.buildLShr(I32, PtrAsInt, Two);
+
+ const auto BitWidthReg = B.buildConstant(I32, MemSize);
+ Register BitOffsetReg;
+ if (HaveConstantBitOffset) {
+ BitOffsetReg = B.buildConstant(I32, ConstantBitOffsetVal).getReg(0);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the low 5 bits of the bit
+ // offset, so masking the high bits is unnecessary.
+ auto Three = B.buildConstant(I32, 3);
+ BitOffsetReg = B.buildShl(I32, PtrAsInt, Three).getReg(0);
+ }
+
+ if (IsStore) {
+ LLT IValTy = LLT::integer(ValSize);
+ Register Value = ValReg;
+ if (IValTy != ValTy)
+ Value = B.buildBitcast(IValTy, Value).getReg(0);
+ if (ValSize != 32)
+ Value = B.buildAnyExt(I32, Value).getReg(0);
+
+ Register ShiftAmt = BitOffsetReg;
+ if (!HaveConstantBitOffset) {
+ auto Mask = B.buildConstant(I32, 31);
+ ShiftAmt = B.buildAnd(I32, BitOffsetReg, Mask).getReg(0);
+ }
+ Value = B.buildShl(I32, Value, ShiftAmt).getReg(0);
+
+ const auto MaskBase = B.buildConstant(I32, (1u << MemSize) - 1);
+ const auto Mask = B.buildShl(I32, MaskBase, ShiftAmt);
+
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE_BITS, {}, {Value, Index, Mask})
+ .addMemOperand(&MMO);
+ } else {
+ const bool IsSExt = Opcode == AMDGPU::G_SEXTLOAD;
+ const auto IsSExtReg = B.buildConstant(I32, IsSExt ? 1 : 0);
+ Register Result =
+ B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD_BITS, {I32},
+ {Index, BitWidthReg, BitOffsetReg, IsSExtReg})
+ .addMemOperand(&MMO)
+ .getReg(0);
+
+ if (ValTy != I32) {
+ LLT ResultTy = I32;
+ if (ValSize != 32) {
+ ResultTy = LLT::integer(ValSize);
+ Result = B.buildTrunc(ResultTy, Result).getReg(0);
+ }
+ if (ResultTy != ValTy)
+ Result = B.buildBitcast(ValTy, Result).getReg(0);
+ }
+
+ B.buildCopy(ValReg, Result);
+ }
+
+ MI.eraseFromParent();
+ return true;
+ }
+
const auto PtrAsInt = B.buildPtrToInt(I32, PtrReg);
auto Two = B.buildConstant(I32, 2);
const auto Index = B.buildLShr(I32, PtrAsInt, Two);
@@ -3552,7 +3664,9 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
- if (AddrSpace == AMDGPUAS::VGPR && MI.getOpcode() == AMDGPU::G_LOAD)
+ // G_LOAD as well as the extending loads (G_SEXTLOAD / G_ZEXTLOAD) are
+ // custom-lowered; the latter only exist for sub-dword accesses.
+ if (AddrSpace == AMDGPUAS::VGPR)
return lowerLoadStoreVGPR(Helper, MI);
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
new file mode 100644
index 0000000000000..95cfe42df7e6d
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -0,0 +1,201 @@
+//===- AMDGPULowerIdxOps.cpp - Expand sub-dword VGPR-memory accesses -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Expand the sub-dword VGPR "as memory" (address space 13) pseudos into a
+/// whole-dword indexed access plus a bit-field extract or insert:
+///
+/// V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+/// V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+///
+/// A sub-dword store is therefore a read-modify-write of the containing dword.
+/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
+/// here take part in the usual M0 setup, and before register allocation
+/// because it introduces new virtual registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-lower-idx-ops"
+
+namespace {
+
+class LowerIdxOps {
+public:
+ LowerIdxOps(MachineFunction &MF)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
+ MRI(&MF.getRegInfo()) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ void lowerLoadIdxBits(MachineInstr &MI);
+ void lowerStoreIdxBits(MachineInstr &MI);
+
+ const SIInstrInfo *TII;
+ MachineRegisterInfo *MRI;
+};
+
+void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
+
+ const bool IsSigned = MI.getOperand(5).getImm() != 0;
+ const MCInstrDesc &II =
+ TII->get(IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPUMI::VLoadIdxInst::getOpcodeForBitWidth(32)),
+ SrcAReg)
+ .add(LoadIdx.getIdxOp())
+ .add(LoadIdx.getOffsetOp());
+ LoadMIB.addMemOperand(*MI.memoperands_begin());
+ // Match what instruction selection does for a whole-dword access with a
+ // register index: record that the M0 write implied by the eventual movrel
+ // clobbers M0 (see AMDGPUAssignIdxToM0).
+ if (LoadIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Extract the accessed bits out of it.
+ MachineOperand BitOffset = MI.getOperand(4);
+ if (BitOffset.isImm())
+ BitOffset.setImm(BitOffset.getImm() & 31);
+
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ .addReg(SrcAReg)
+ .add(BitOffset)
+ .add(MI.getOperand(3)); // bitsize
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineFunction *MF = MBB->getParent();
+ auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+
+ Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Read the containing dword.
+ auto LoadMIB = BuildMI(*MBB, MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::V_LOAD_IDX_B32), SrcAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ // The index is read again by the store below, so it does not die here.
+ LoadMIB->getOperand(1).setIsKill(false);
+ auto *StoreMMO = *MI.memoperands_begin();
+ // Synthesize a load MMO from the store's.
+ auto NewFlags = MachineMemOperand::MOLoad;
+ NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
+ LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ if (StoreIdx.getIdxOp().isReg())
+ LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ // Insert the stored bits into it.
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
+ .add(StoreIdx.getOperand(3)) // mask
+ .addReg(StoreIdx.getDataOp().getReg())
+ .addReg(SrcAReg);
+
+ // Write the dword back.
+ auto StoreMIB =
+ BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_STORE_IDX_B32))
+ .addReg(DstAReg)
+ .add(StoreIdx.getIdxOp())
+ .add(StoreIdx.getOffsetOp());
+ StoreMIB.addMemOperand(StoreMMO);
+ if (StoreIdx.getIdxOp().isReg())
+ StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+
+ LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
+
+ MI.eraseFromParent();
+}
+
+bool LowerIdxOps::run(MachineFunction &MF) {
+ bool Changed = false;
+
+ LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (auto MII = MBB.begin(), E = MBB.end(); MII != E;) {
+ MachineInstr &MI = *MII++;
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_LOAD_IDX_BITS:
+ lowerLoadIdxBits(MI);
+ Changed = true;
+ break;
+ case AMDGPU::V_STORE_IDX_BITS:
+ lowerStoreIdxBits(MI);
+ Changed = true;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ return Changed;
+}
+
+class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return LowerIdxOps(MF).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "AMDGPU Lower Idx Ops"; }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!LowerIdxOps(MF).run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char AMDGPULowerIdxOpsLegacy::ID = 0;
+
+char &llvm::AMDGPULowerIdxOpsID = AMDGPULowerIdxOpsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPULowerIdxOpsLegacy, DEBUG_TYPE, "AMDGPU Lower Idx Ops",
+ false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
index 697c63e2079d1..970e9916f14dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.cpp
@@ -19,14 +19,19 @@ using namespace llvm;
using namespace AMDGPUMI;
unsigned VLoadStoreIdxInst::getBitWidth() const {
+ unsigned Opc = getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ llvm_unreachable("V_LOAD/STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
if (!Info)
llvm_unreachable("unsupported V_LOAD/STORE_IDX opcode");
return Info->BitWidth;
}
int VLoadIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_LOAD_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/false);
if (!Info)
@@ -41,6 +46,8 @@ unsigned VLoadIdxInst::getOpcodeForBitWidth(unsigned Bits) {
}
int VStoreIdxInst::tryGetOpcodeForBitWidth(unsigned Bits) {
+ if (Bits == 8 || Bits == 16)
+ llvm_unreachable("V_STORE_IDX_BITS has no well defined bit width");
const AMDGPU::VLdStIdxOpcodeInfo *Info =
AMDGPU::getVLdStIdxOpcodeInfoByKey(Bits, /*IsStore=*/true);
if (!Info)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index 24e24aab53152..e99594062871a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -37,7 +37,10 @@ class VLoadStoreIdxInst : public MachineInstr {
unsigned getBitWidth() const;
static bool classof(const MachineInstr *MI) {
- return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode()) != nullptr;
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS || Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
+ return AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc) != nullptr;
}
};
@@ -47,8 +50,11 @@ class VLoadIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_LOAD_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && !Info->IsStore;
}
};
@@ -59,8 +65,11 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
static unsigned getOpcodeForBitWidth(unsigned Bits);
static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ if (Opc == AMDGPU::V_STORE_IDX_BITS)
+ return true;
const AMDGPU::VLdStIdxOpcodeInfo *Info =
- AMDGPU::getVLdStIdxOpcodeInfoByOpcode(MI->getOpcode());
+ AMDGPU::getVLdStIdxOpcodeInfoByOpcode(Opc);
return Info && Info->IsStore;
}
};
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d0255025298d7..712197b54b9fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -116,6 +116,7 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
MACHINE_FUNCTION_PASS("amdgpu-assign-idx-to-m0", AMDGPUAssignIdxToM0Pass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
+MACHINE_FUNCTION_PASS("amdgpu-lower-idx-ops", AMDGPULowerIdxOpsPass())
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 6c426c6da11e8..3428047645339 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1387,6 +1387,12 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForGOpcs({G_AMDGPU_REG_STORE})
.Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF}}});
+ addRulesForGOpcs({G_AMDGPU_REG_LOAD_BITS})
+ .Any({{BRC}, {{VgprBRC}, {Sgpr32_WF, Sgpr32, Vgpr32, Sgpr32}}});
+
+ addRulesForGOpcs({G_AMDGPU_REG_STORE_BITS})
+ .Any({{BRC}, {{}, {VgprBRC, Sgpr32_WF, Vgpr32}}});
+
addRulesForGOpcs({G_PTR_ADD})
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 5bdf0768179ec..e672498a75941 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3088,7 +3088,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
return;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// The dword index (operand 1) must be uniform; a divergent index needs a
// waterfall loop.
applyDefaultMapping(OpdMapper);
@@ -4495,10 +4497,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
break;
}
case AMDGPU::G_AMDGPU_REG_LOAD:
- case AMDGPU::G_AMDGPU_REG_STORE: {
+ case AMDGPU::G_AMDGPU_REG_STORE:
+ case AMDGPU::G_AMDGPU_REG_LOAD_BITS:
+ case AMDGPU::G_AMDGPU_REG_STORE_BITS: {
// data/result is a VGPR value; the dword index is uniform (SGPR).
OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ const unsigned Opc = MI.getOpcode();
+ if (Opc == AMDGPU::G_AMDGPU_REG_LOAD_BITS) {
+ // {dst, index, bitsize, bitoffset, is_sext_load}
+ OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+ OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+ } else if (Opc == AMDGPU::G_AMDGPU_REG_STORE_BITS) {
+ // {value, index, mask}
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ }
break;
}
case AMDGPU::G_AMDGPU_BUFFER_LOAD:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index a5dadbf76675a..45b5f9bd1d1dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -669,6 +669,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
initializeAMDGPUAssignIdxToM0LegacyPass(*PR);
+ initializeAMDGPULowerIdxOpsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1780,6 +1781,11 @@ void GCNPassConfig::addFastRegAlloc() {
}
void GCNPassConfig::addPreRegAlloc() {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert, so that the M0 setup below sees the
+ // indexed accesses this creates.
+ addPass(&AMDGPULowerIdxOpsID);
+
// Copy the VGPR "as memory" load/store index into M0 before register
// allocation; the movrel emitted later by AMDGPULowerVGPREncoding reads it.
addPass(&AMDGPUAssignIdxToM0ID);
@@ -2600,6 +2606,10 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
}
void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
+ // Expand the sub-dword VGPR "as memory" pseudos into a whole-dword access
+ // plus a bit-field extract/insert, before the M0 setup below.
+ addMachineFunctionPass(AMDGPULowerIdxOpsPass(), PMW);
+
// Set up M0 for the movrel that expands a VGPR "as memory" indexed access.
// Run before allocation so the index computation coalesces into M0.
addMachineFunctionPass(AMDGPUAssignIdxToM0Pass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 054fc1d14d7cb..8aaed3e8af56d 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -91,6 +91,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUMemoryUtils.cpp
AMDGPUCoExecSchedStrategy.cpp
AMDGPUIGroupLP.cpp
+ AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ebeebbf9e01a3..f505c2bcccf1a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13185,17 +13185,17 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Only whole-dword, non-extending/non-truncating accesses are implemented.
- // Reject anything else with a diagnostic (replacing the value with poison)
- // instead of failing instruction selection. Both callers - operation
- // legalization and the pre-ISel combine - replace the node with this result,
- // so the diagnostic is emitted exactly once.
+ // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
+ // with a diagnostic (replacing the value with poison) instead of failing
+ // instruction selection. Both callers - operation legalization and the
+ // pre-ISel combine - replace the node with this result, so the diagnostic is
+ // emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword loads and stores are implemented",
+ "whole-dword and 8-/16-bit loads and stores are implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13203,8 +13203,112 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
{DAG.getPOISON(Op.getValueType()), MemOp->getChain()}, DL);
};
- if (BitWidth < 32)
- return reportUnsupported();
+ // Handle bytes and aligned shorts. These become a bit-field extract out of
+ // the containing dword (loads), or a read-modify-write of it (stores); see
+ // AMDGPULowerIdxOps.
+ if (BitWidth < 32) {
+ if (BitWidth != 8 && BitWidth != 16)
+ return reportUnsupported();
+
+ // Bail out for sub-dword types we cannot handle.
+ if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
+ if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
+ Load->getValueType(0).getSizeInBits() > 32)
+ return reportUnsupported();
+ } else {
+ auto *Store = cast<StoreSDNode>(MemOp);
+ if (Store->isTruncatingStore() &&
+ Store->getValue().getValueType() != MVT::i16 &&
+ Store->getValue().getValueType() != MVT::i32)
+ return reportUnsupported();
+ }
+
+ SDValue Ptr = MemOp->getBasePtr();
+
+ // Split the address into base and constant offset.
+ SDValue Base;
+ int32_t Offset;
+ if (DAG.isBaseWithConstantOffset(Ptr)) {
+ Base = Ptr.getOperand(0);
+ Offset = cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue();
+ } else if (auto *C = dyn_cast<ConstantSDNode>(Ptr)) {
+ Base = DAG.getConstant(0, DL, MVT::i32);
+ Offset = C->getSExtValue();
+ } else {
+ Base = Ptr;
+ Offset = 0;
+ }
+
+ // Determine the bit-offset, optimizing the case where the LSBs are
+ // constant.
+ KnownBits BaseKB;
+ if (Offset == 0 && MemOp->getAlign() >= Align(4)) {
+ BaseKB = KnownBits::makeConstant(APInt::getZero(2));
+ } else {
+ BaseKB = DAG.computeKnownBits(Base).trunc(2);
+ }
+
+ SDValue BitOffset;
+ if (BaseKB.isConstant()) {
+ Offset += BaseKB.getConstant().getZExtValue();
+ BitOffset = DAG.getConstant((Offset & 3) * 8, DL, MVT::i32);
+ } else {
+ // V_{LOAD,STORE}_IDX_BITS only care about the least 5 bits of the bit
+ // offset, so we do not have to mask off the high bits.
+ BitOffset = DAG.getNode(ISD::SHL, DL, MVT::i32, Ptr,
+ DAG.getConstant(3, DL, MVT::i32));
+ }
+
+ SDValue Index = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
+ DAG.getConstant(2, DL, MVT::i32));
+ SDValue BitSizeImm = DAG.getConstant(BitWidth, DL, MVT::i32);
+ SDValue Chain = MemOp->getChain();
+
+ if (auto *StoreOp = dyn_cast<StoreSDNode>(MemOp)) {
+ SDValue Value = StoreOp->getValue();
+ EVT ValVT = Value.getValueType();
+ if (!ValVT.isScalarInteger())
+ Value = DAG.getNode(
+ ISD::BITCAST, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ValVT.getSizeInBits()), Value);
+ Value = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Value);
+
+ // ISD::SHL produces poison for shift amounts >= bit width, which generic
+ // DAG combines rely on. We need the mask for correctness, but it should
+ // get optimized away during final instruction selection.
+ SDValue ShiftOffset = DAG.getNode(ISD::AND, DL, MVT::i32, BitOffset,
+ DAG.getConstant(31, DL, MVT::i32));
+ Value = DAG.getNode(ISD::SHL, DL, MVT::i32, Value, ShiftOffset);
+
+ SDValue MaskBase = DAG.getConstant((1u << BitWidth) - 1, DL, MVT::i32);
+ SDValue Mask = DAG.getNode(ISD::SHL, DL, MVT::i32, MaskBase, ShiftOffset);
+
+ return DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_STORE_BITS, DL, DAG.getVTList(MVT::Other),
+ {Chain, Value, Index, Mask}, MemVT, StoreOp->getMemOperand());
+ }
+
+ auto *LoadOp = cast<LoadSDNode>(MemOp);
+ bool IsSExt = LoadOp->getExtensionType() == ISD::SEXTLOAD;
+ SDValue IsSExtImm = DAG.getConstant(IsSExt ? 1 : 0, DL, MVT::i32);
+ SDValue Value = DAG.getMemIntrinsicNode(
+ AMDGPUISD::REG_LOAD_BITS, DL, DAG.getVTList(MVT::i32, MVT::Other),
+ {Chain, Index, BitSizeImm, BitOffset, IsSExtImm}, MemVT,
+ LoadOp->getMemOperand());
+ EVT ResVT = LoadOp->getValueType(0);
+ if (ResVT == MVT::i32)
+ return Value;
+
+ SDValue LoadChain = Value.getValue(1);
+ unsigned ResWidth = ResVT.getSizeInBits();
+ if (ResWidth < 32)
+ Value =
+ DAG.getNode(ISD::TRUNCATE, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ResWidth), Value);
+ if (Value.getValueType() != ResVT)
+ Value = DAG.getNode(ISD::BITCAST, DL, ResVT, Value);
+ return DAG.getMergeValues({Value, LoadChain}, DL);
+ }
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD)
return reportUnsupported();
@@ -18881,7 +18985,9 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::FP_ROUND:
return performFPRoundCombine(N, DCI);
case AMDGPUISD::REG_LOAD:
- case AMDGPUISD::REG_STORE: {
+ case AMDGPUISD::REG_STORE:
+ case AMDGPUISD::REG_LOAD_BITS:
+ case AMDGPUISD::REG_STORE_BITS: {
const SIMachineFunctionInfo *MFI =
DCI.DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
unsigned NumAddressableVGPRs =
@@ -18889,7 +18995,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0;
+ unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
@@ -18897,10 +19003,22 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case AMDGPUISD::REG_STORE:
IndexOpIdx = 2;
break;
+ case AMDGPUISD::REG_LOAD_BITS:
+ IndexOpIdx = 1;
+ BitOffOpIdx = 3;
+ break;
+ case AMDGPUISD::REG_STORE_BITS:
+ IndexOpIdx = 2;
+ break;
}
if (SimplifyDemandedBits(N->getOperand(IndexOpIdx), IndexMask, DCI))
return SDValue(N, 0);
+ if (BitOffOpIdx) {
+ APInt BitOffMask = APInt::getLowBitsSet(32, 5);
+ if (SimplifyDemandedBits(N->getOperand(BitOffOpIdx), BitOffMask, DCI))
+ return SDValue(N, 0);
+ }
break;
}
case ISD::LOAD: {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 1d9a86f7fa4a4..f9783e7a82115 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -11012,7 +11012,8 @@ SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
// A VGPR ("as memory") indexed load is always divergent: it reads the wave's
// per-lane view of its vector registers, so even a uniform index yields a
// per-lane (divergent) value.
- if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
+ if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD ||
+ Opcode == AMDGPU::G_AMDGPU_REG_LOAD_BITS)
return ValueUniformity::NeverUniform;
// Loads from the private and flat address spaces are divergent, because
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c6bbfc1ec14b4..0b72f2f0c021d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -70,6 +70,24 @@ def SIreg_load : SDNode<"AMDGPUISD::REG_LOAD", SDTRegIdxLoad,
def SIreg_store : SDNode<"AMDGPUISD::REG_STORE", SDTRegIdxStore,
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+// VGPR address space sub-dword (8/16-bit) load/store with dword index. The
+// access is described by the dword index plus the bit position within that
+// dword; the store form carries a pre-shifted value and an insert mask.
+def SDTRegIdxBitsLoad : SDTypeProfile<1, 4,
+ [SDTCisInt<0>, // result (i32 or i16)
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>, // bitsize
+ SDTCisVT<3, i32>, // bitoffset
+ SDTCisVT<4, i32>]>; // is_sext_load
+def SDTRegIdxBitsStore : SDTypeProfile<0, 3,
+ [SDTCisInt<0>, // data (i32 or i16), pre-shifted insert value
+ SDTCisVT<1, i32>, // dword_index
+ SDTCisVT<2, i32>]>; // insert mask
+def SIreg_load_bits : SDNode<"AMDGPUISD::REG_LOAD_BITS", SDTRegIdxBitsLoad,
+ [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def SIreg_store_bits : SDNode<"AMDGPUISD::REG_STORE_BITS", SDTRegIdxBitsStore,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
def SDTSBufferLoad : SDTypeProfile<1, 3,
[ // vdata
SDTCisVT<1, v4i32>, // rsrc
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f62e77e6c740b..6a30652085cb0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1160,6 +1160,70 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+// Used for byte or short loads.
+// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
+// sign-extended loads.
+//
+// Only the least 5 bits of $bitoffset are significant, the rest are ignored.
+// Only offset values of 0, 8, 16, or 24 are meaningful (after masking to the
+// 5 LSBs).
+//
+// Only bitsize of 8 or 16 bits are supported and must be a compile-time
+// constant.
+class V_LOAD_IDX_BITS<RegisterClass data_rc>
+ : VPseudoInstSI<(outs data_rc:$data),
+ (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
+ VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+ let mayLoad = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
+
+// Used for byte or short stores.
+//
+// $data holds the pre-shifted insert value and $mask holds the insert mask
+// that is used when lowering to V_BFI_B32.
+class V_STORE_IDX_BITS<RegisterClass data_op_rc>
+ : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
+ let mayStore = 1;
+ let VALU = 1;
+ let UseNamedOperandTable = 1;
+ let hasSideEffects = 0;
+ let Defs = [M0];
+}
+
+def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
+
+// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
+// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
+// shape folds a constant dword offset into the pseudo's $offset operand.
+multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
+ Instruction store_inst, ValueType vt> {
+ def : GCNPat<
+ (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+ def : GCNPat<
+ (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (store_inst $data, $idx, imm:$offset, $mask)>;
+ def : GCNPat<
+ (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
+ (store_inst $data, $idx, 0, $mask)>;
+}
+
+defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
// pseudo avoids problems with the Two Address instructions pass.
@@ -4937,6 +5001,24 @@ def G_AMDGPU_REG_STORE : AMDGPUGenericInstruction {
let mayStore = 1;
}
+// Sub-dword (8/16-bit) counterparts of the above. In addition to the dword
+// index they carry the bit position of the access within that dword, and are
+// lowered to a read-modify-write of the whole dword (see AMDGPULowerIdxOps).
+def G_AMDGPU_REG_LOAD_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$dword_index, type1:$bitsize,
+ type1:$bitoffset, type1:$is_sext_load);
+ let hasSideEffects = 0;
+ let mayLoad = 1;
+}
+
+def G_AMDGPU_REG_STORE_BITS : AMDGPUGenericInstruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins type0:$data, type1:$dword_index, type1:$mask);
+ let hasSideEffects = 0;
+ let mayStore = 1;
+}
+
def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type0:$src0, type0:$src1);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
new file mode 100644
index 0000000000000..31c9b44e7236d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -0,0 +1,449 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
+; address space (13). A load reads the containing dword with an M0-relative
+; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
+; read-modify-write of that dword using v_bfi_b32 (see AMDGPULowerIdxOps).
+
+define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ ret i8 %x
+}
+
+define i32 @load_i8_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i8_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = sext i8 %x to i32
+ ret i32 %y
+}
+
+define i16 @load_i16(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ ret i16 %x
+}
+
+define i32 @load_i16_zext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_zext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+define i32 @load_i16_sext(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i16_sext:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_i32 v0, v0, s0, 16
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i16, ptr addrspace(13) %p
+ %y = sext i16 %x to i32
+ ret i32 %y
+}
+
+; A dword-aligned access has a statically known bit offset of zero.
+define i32 @load_i8_aligned(ptr addrspace(13) inreg %p) {
+; GFX12-LABEL: load_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v0, v0, 0, 8
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+; A constant byte offset folds into a constant bit offset.
+define i32 @load_i8_const_offset(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i8_const_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 6
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_const_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 6
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 6
+ %x = load i8, ptr addrspace(13) %q, align 2
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
+; GFX12-SDAG-LABEL: store_i16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i16 %v, ptr addrspace(13) %p
+ ret void
+}
+
+define void @store_i8_aligned(ptr addrspace(13) inreg %p, i8 %v) {
+; GFX12-LABEL: store_i8_aligned:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-NEXT: s_mov_b32 m0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_bfi_b32 v0, 0xff, v0, v1
+; GFX12-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; A divergent index is made uniform by a waterfall loop.
+define i32 @load_i8_divergent(ptr addrspace(13) %p) {
+; GFX12-SDAG-LABEL: load_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, v1, 8
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB11_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
+; GFX12-SDAG-LABEL: store_i8_divergent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 s1, s0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-SDAG-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s1, s1
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX12-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-SDAG-NEXT: ; %bb.2:
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_divergent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 31, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 2, v0
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v3, 0xff
+; GFX12-GISEL-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v0, v1, v2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB12_1
+; GFX12-GISEL-NEXT: ; %bb.2:
+; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 40263e1c975ad..058dd5f1c6e0c 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,31 +1,36 @@
; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
-; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13) are
-; not yet implemented. They must be rejected with a clean diagnostic on both
-; SelectionDAG and GlobalISel, rather than failing with "cannot select" /
-; "unable to legalize".
+; Accesses of the VGPR "as memory" address space (13) that are still not
+; implemented must be rejected with a clean diagnostic on both SelectionDAG and
+; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
+; Whole-dword and 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
+; as-vgpr-bits.ll.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define i8 @load_i8(ptr addrspace(13) inreg %p) {
+; A sub-dword load extended into a value wider than a dword.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
- ret i8 %x
+ %y = zext i8 %x to i64
+ ret i64 %y
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define i16 @load_i16(ptr addrspace(13) inreg %p) {
- %x = load i16, ptr addrspace(13) %p
- ret i16 %x
+; A memory size that is neither a whole dword nor 8/16 bits.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define i1 @load_i1(ptr addrspace(13) inreg %p) {
+ %x = load i1, ptr addrspace(13) %p
+ ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define void @store_i8(ptr addrspace(13) inreg %p, i8 %v) {
- store i8 %v, ptr addrspace(13) %p
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
+ store i1 %v, ptr addrspace(13) %p
ret void
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword loads and stores are implemented
-define void @store_i16(ptr addrspace(13) inreg %p, i16 %v) {
- store i16 %v, ptr addrspace(13) %p
- ret void
+; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
+ %x = load <14 x i32>, ptr addrspace(13) %p
+ ret <14 x i32> %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 435ea17e77122..d2418eabdb0df 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -69,6 +69,7 @@
; GCN-O0-NEXT: function
; GCN-O0-NEXT: machine-function
; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: amdgpu-assign-idx-to-m0
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
@@ -218,6 +219,7 @@
; GCN-O2-NEXT: function
; GCN-O2-NEXT: machine-function
; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-lower-idx-ops
; GCN-O2-NEXT: amdgpu-assign-idx-to-m0
; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O2-NEXT: detect-dead-lanes
@@ -407,6 +409,7 @@
; GCN-O3-NEXT: function
; GCN-O3-NEXT: machine-function
; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-lower-idx-ops
; GCN-O3-NEXT: amdgpu-assign-idx-to-m0
; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
; GCN-O3-NEXT: detect-dead-lanes
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 8a1be666cb68a..85dd7dfee0f2e 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -113,6 +113,7 @@
; GCN-O0-NEXT: Finalize ISel and expand pseudo-instructions
; GCN-O0-NEXT: Local Stack Slot Allocation
; GCN-O0-NEXT: Register Usage Information Propagation
+; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: AMDGPU Assign Idx To M0
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
@@ -352,6 +353,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
@@ -675,6 +677,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Lower Idx Ops
; GCN-O1-OPTS-NEXT: AMDGPU Assign Idx To M0
; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
@@ -1002,6 +1005,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Lower Idx Ops
; GCN-O2-NEXT: AMDGPU Assign Idx To M0
; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
@@ -1343,6 +1347,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Lower Idx Ops
; GCN-O3-NEXT: AMDGPU Assign Idx To M0
; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
>From df5a87d175df55ac0db392c1cacdcb31327182aa Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 13:14:32 +0300
Subject: [PATCH 02/15] Correctly diagnose the mis-aligned sub-dowrd accesses
instead of miscompiling them
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 27 ++++++++----
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 19 +++++---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 43 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-unsupported.ll | 33 ++++++++++----
4 files changed, 99 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 230cdd2a19c02..c6acb27e31d38 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -556,11 +556,21 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
}
// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value. Whole-dword accesses
-// (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo) are supported, as are
-// 8-/16-bit accesses, including extending loads into a 16- or 32-bit value.
-static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
+// memory access producing/consuming a ValSize-bit value at the given alignment.
+// Whole-dword accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo)
+// are supported, as are 8-/16-bit accesses, including extending loads into a
+// 16- or 32-bit value.
+//
+// A sub-dword access is implemented as a bit-field extract from (or insert
+// into) the dword containing it, so it must not straddle a dword boundary. An
+// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
+// natural alignment covers both, and is what lets the bit offset within the
+// dword be computed from a possibly dynamic pointer.
+static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment) {
if (MemSize == 8 || MemSize == 16) {
+ if (Alignment < Align(MemSize / 8))
+ return false;
if (ValSize == MemSize)
return true;
if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
@@ -3511,14 +3521,15 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// integer types rather than plain scalars.
const LLT I32 = LLT::integer(32);
- // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
- // with a diagnostic instead of failing to legalize.
- if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize)) {
+ // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
+ // Reject anything else with a diagnostic instead of failing to legalize.
+ if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and 8-/16-bit loads and stores are implemented",
+ "whole-dword and naturally aligned 8-/16-bit loads and stores are "
+ "implemented",
MI.getDebugLoc()));
if (!IsStore)
B.buildUndef(ValReg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f505c2bcccf1a..b898b37440b85 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13185,17 +13185,18 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
EVT MemVT = MemOp->getMemoryVT();
unsigned BitWidth = MemVT.getSizeInBits();
- // Whole-dword and 8-/16-bit accesses are implemented. Reject anything else
- // with a diagnostic (replacing the value with poison) instead of failing
- // instruction selection. Both callers - operation legalization and the
- // pre-ISel combine - replace the node with this result, so the diagnostic is
- // emitted exactly once.
+ // Whole-dword and naturally aligned 8-/16-bit accesses are implemented.
+ // Reject anything else with a diagnostic (replacing the value with poison)
+ // instead of failing instruction selection. Both callers - operation
+ // legalization and the pre-ISel combine - replace the node with this result,
+ // so the diagnostic is emitted exactly once.
auto reportUnsupported = [&]() -> SDValue {
const Function &F = DAG.getMachineFunction().getFunction();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
F,
"unsupported access of VGPR 'as memory' address space (13); only "
- "whole-dword and 8-/16-bit loads and stores are implemented",
+ "whole-dword and naturally aligned 8-/16-bit loads and stores are "
+ "implemented",
DL.getDebugLoc()));
if (isa<StoreSDNode>(MemOp))
return MemOp->getChain();
@@ -13210,6 +13211,12 @@ SDValue SITargetLowering::LowerLoadStoreVGPR(SDValue Op,
if (BitWidth != 8 && BitWidth != 16)
return reportUnsupported();
+ // The access becomes a bit-field extract from (or insert into) the dword
+ // containing it, so it must not straddle a dword boundary. An 8-bit access
+ // never can; a 16-bit one only if it is 2-byte aligned.
+ if (MemOp->getAlign() < Align(BitWidth / 8))
+ return reportUnsupported();
+
// Bail out for sub-dword types we cannot handle.
if (auto *Load = dyn_cast<LoadSDNode>(MemOp)) {
if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 31c9b44e7236d..4b1703c1b6043 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -447,3 +447,46 @@ define void @store_i8_divergent(ptr addrspace(13) %p, i8 %v) {
store i8 %v, ptr addrspace(13) %p
ret void
}
+
+; A 2-byte aligned 16-bit access sits at bit offset 0 or 16, so it always fits
+; within the containing dword even though the offset is not known statically.
+; (An under-aligned 16-bit access is rejected; see as-vgpr-unsupported.ll.)
+define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
+; GFX12-SDAG-LABEL: load_i16_offset2:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i16_offset2:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 16
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 2
+ %x = load i16, ptr addrspace(13) %q, align 2
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
index 058dd5f1c6e0c..eaf10f59b04f4 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-unsupported.ll
@@ -1,14 +1,29 @@
; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck %s
-; Accesses of the VGPR "as memory" address space (13) that are still not
-; implemented must be rejected with a clean diagnostic on both SelectionDAG and
-; GlobalISel, rather than failing with "cannot select" / "unable to legalize".
-; Whole-dword and 8-/16-bit accesses are implemented; see as-vgpr-basic.ll and
-; as-vgpr-bits.ll.
+; Accesses of the VGPR "as memory" address space (13) that are not implemented
+; must be rejected with a clean diagnostic on both SelectionDAG and GlobalISel,
+; rather than failing with "cannot select" / "unable to legalize" - or, worse,
+; silently generating wrong code. Whole-dword and naturally aligned 8-/16-bit
+; accesses are implemented; see as-vgpr-basic.ll and as-vgpr-bits.ll.
+
+; An under-aligned 16-bit access may straddle a dword boundary, which the
+; bit-field extract / insert lowering cannot express.
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define i32 @load_i16_align1(ptr addrspace(13) inreg %p) {
+ %x = load i16, ptr addrspace(13) %p, align 1
+ %y = zext i16 %x to i32
+ ret i32 %y
+}
+
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
+define void @store_i16_align1(ptr addrspace(13) inreg %p, i16 %v) {
+ store i16 %v, ptr addrspace(13) %p, align 1
+ ret void
+}
; A sub-dword load extended into a value wider than a dword.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
%x = load i8, ptr addrspace(13) %p
%y = zext i8 %x to i64
@@ -16,20 +31,20 @@ define i64 @load_i8_zext_i64(ptr addrspace(13) inreg %p) {
}
; A memory size that is neither a whole dword nor 8/16 bits.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define i1 @load_i1(ptr addrspace(13) inreg %p) {
%x = load i1, ptr addrspace(13) %p
ret i1 %x
}
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define void @store_i1(ptr addrspace(13) inreg %p, i1 %v) {
store i1 %v, ptr addrspace(13) %p
ret void
}
; A whole-dword size with no corresponding V_LOAD_IDX/V_STORE_IDX pseudo.
-; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and 8-/16-bit loads and stores are implemented
+; CHECK: error: {{.*}}unsupported access of VGPR 'as memory' address space (13); only whole-dword and naturally aligned 8-/16-bit loads and stores are implemented
define <14 x i32> @load_v14i32(ptr addrspace(13) inreg %p) {
%x = load <14 x i32>, ptr addrspace(13) %p
ret <14 x i32> %x
>From e65cc336ed164d5d4065a056b0e1c04bd0d70b51 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:12:39 +0300
Subject: [PATCH 03/15] Run AMDGPULowerIdxOps for optnone functions in
AMDGPULowerIdxOps
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 5 +-
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 110 ++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 95cfe42df7e6d..e7d3a1e8bf172 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -168,8 +168,9 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
AMDGPULowerIdxOpsLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (skipFunction(MF.getFunction()))
- return false;
+ // This is required lowering, not an optimization: nothing else expands the
+ // sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
+ // therefore must not be skipped for optnone functions.
return LowerIdxOps(MF).run(MF);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 4b1703c1b6043..66790a4a155fe 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -490,3 +490,113 @@ define i32 @load_i16_offset2(ptr addrspace(13) inreg %p) {
%y = zext i16 %x to i32
ret i32 %y
}
+
+; Expanding these pseudos is required lowering, so it has to happen for optnone
+; functions too - clang marks every function optnone at -O0. Nothing else can
+; lower them, so if the expansion is skipped AMDGPULowerVGPREncoding is left
+; with a pseudo it cannot handle.
+define i32 @load_i8_optnone(ptr addrspace(13) inreg %p) noinline optnone {
+; GFX12-SDAG-LABEL: load_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, s2
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s1, 8
+; GFX12-SDAG-NEXT: s_mov_b32 s0, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_and_b32_e64 v0, v0, s0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v1, v0, 8
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %x = load i8, ptr addrspace(13) %p
+ %y = zext i8 %x to i32
+ ret i32 %y
+}
+
+define void @store_i8_optnone(ptr addrspace(13) inreg %p, i8 %v) noinline optnone {
+; GFX12-SDAG-LABEL: store_i8_optnone:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ; kill: def $vgpr1_lo16 killed $vgpr0 killed $exec
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e64 v0, s1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s2, 0xff
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, s2, s1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_i8_optnone:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshr_b32 s1, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, s2
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 31
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s0, s2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e64 v0, v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0xff
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s2, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, v1, v0, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
>From c054c1c9a09156db36ec9a4d83c144212a5536fe Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:50:21 +0300
Subject: [PATCH 04/15] Add a MIR test for AMDGPULowerIdxOps
---
.../as-vgpr-lower-idx-ops.mir | 134 ++++++++++++++++++
1 file changed, 134 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
new file mode 100644
index 0000000000000..c991990db933e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=amdgpu-lower-idx-ops -o - %s | FileCheck %s
+
+# Expansion of the sub-dword VGPR "as memory" (address space 13) pseudos by
+# AMDGPULowerIdxOps, in isolation:
+#
+# V_LOAD_IDX_BITS -> V_LOAD_IDX_B32 + V_BFE_{U,I}32
+# V_STORE_IDX_BITS -> V_LOAD_IDX_B32 + V_BFI_B32 + V_STORE_IDX_B32
+#
+# The pass runs before AMDGPUAssignIdxToM0, so a register index is still
+# virtual here and the implicit-def $m0 that instruction selection attached is
+# carried over to the whole-dword accesses the expansion creates.
+
+---
+name: lower_load_idx_bits_zext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_zext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_sext
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_sext
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 1, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: lower_load_idx_bits_16_offset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_16_offset
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = V_LOAD_IDX_BITS %0, 3, 16, 16, 0, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ S_ENDPGM 0, implicit %1
+...
+
+# A dynamic bit offset is passed straight through to the bit-field extract;
+# only its low 5 bits are significant.
+---
+name: lower_load_idx_bits_dynamic_bitoffset
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_load_idx_bits_dynamic_bitoffset
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, %1, 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2
+...
+
+# The store is a read-modify-write of the containing dword. The index is read
+# again by the store, so it must not be killed by the load.
+---
+name: lower_store_idx_bits
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ ; CHECK-LABEL: name: lower_store_idx_bits
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ V_STORE_IDX_BITS %1, killed %0, 0, 255, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ S_ENDPGM 0
+...
+
+---
+name: lower_store_idx_bits_dynamic_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: lower_store_idx_bits_dynamic_mask
+ ; CHECK: liveins: $sgpr0, $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = COPY $vgpr1
+ V_STORE_IDX_BITS %1, %0, 2, %2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ S_ENDPGM 0
+...
>From 085ba4b642b360135704c22f0f573b331870e9e0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:51:08 +0300
Subject: [PATCH 05/15] Verify operand invariants of the VGPR-memory indexed
accesses
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 28 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 26 +++++++++++++++++
2 files changed, 54 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f9783e7a82115..c7c40af261b76 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5218,6 +5218,34 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
+ // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
+ // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
+ // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
+ // access has to record that the move clobbers M0, otherwise it could be
+ // separated from the code that sets M0 up.
+ if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
+ if (MI.getNumMemOperands() != 1) {
+ ErrInfo = "v_load/store_idx should have exactly one memory operand.";
+ return false;
+ }
+
+ const MachineOperand &IdxOp = LdStIdx->getIdxOp();
+ if (IdxOp.isReg()) {
+ if (IdxOp.getSubReg() != 0) {
+ ErrInfo =
+ "v_load/store_idx register index must not have a subregister.";
+ return false;
+ }
+
+ if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
+ !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
+ ErrInfo = "v_load/store_idx with register index must have implicit-def "
+ "$m0 on movrel subtargets.";
+ return false;
+ }
+ }
+ }
+
// Make sure the register classes are correct.
for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
new file mode 100644
index 0000000000000..bec5ef54aa203
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -0,0 +1,26 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
+# access ends up in M0, which the v_movrel[sd] emitted by
+# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
+# index to M0, the access must record that the eventual move clobbers M0 -
+# otherwise it could be separated from the code that sets M0 up. Check that the
+# machine verifier rejects a register index without that implicit-def, for both
+# the whole-dword and the sub-dword pseudos.
+
+# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
+
+---
+name: missing_m0_implicit_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
+ %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
+ S_ENDPGM 0, implicit %2, implicit %3
+...
>From 39b9d7d3a5e4b71a489944e2bebcea638f6fb9e0 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 14:59:26 +0300
Subject: [PATCH 06/15] Test the bit-offset half of the VGPR-memory
demanded-bits combine
---
.../as-vgpr-index-demanded-bits.ll | 79 +++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 1a2689f23a1c4..5ce932cb75200 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -7,6 +7,11 @@
; folds away via the AMDGPUISD::REG_LOAD / REG_STORE SimplifyDemandedBits combine.
; The incoming index is masked with 0xffff (wider than necessary); on the SDAG
; path the mask must not survive into the M0 index computation.
+;
+; The same combine also trims the bit offset of a sub-dword access, which needs
+; only its low 5 bits to name a bit position within the containing dword. That
+; half matters independently: a mask can be dead for the bit offset while the
+; index computation keeps it alive, and vice versa.
define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-SDAG-LABEL: load_masked_index:
@@ -47,5 +52,79 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
store i32 %val, ptr addrspace(13) %ptr
ret void
}
+
+; A dynamic pointer masked with 0x3ff: the mask is dead both for the index and
+; for the bit offset, so no s_and may remain.
+define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %m = and i32 %sum, 1023
+ %ptr = inttoptr i32 %m to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
+
+; Setting a high bit of the pointer cannot affect the 5-bit bit offset either,
+; so no bit-set of the shifted offset may remain.
+define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_bits_or_high_bit:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: load_bits_or_high_bit:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-NEXT: ; return to shader part epilog
+ %sum = add i32 %arg, %n
+ %o = or i32 %sum, 65536
+ %ptr = inttoptr i32 %o to ptr addrspace(13)
+ %v = load i8, ptr addrspace(13) %ptr
+ %z = zext i8 %v to i32
+ ret i32 %z
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
>From 3068b32c508e0fe55f29dc8ce3aa0ba06fa6661f Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:13:46 +0300
Subject: [PATCH 07/15] Simplify the sub-dword VGPR-memory pseudo definitions
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 57 +++++++++++-------------
1 file changed, 25 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 6a30652085cb0..79e35a19a9f72 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1170,10 +1170,12 @@ defm : VRegIdxLoadStorePat<vt>;
//
// Only bitsize of 8 or 16 bits are supported and must be a compile-time
// constant.
-class V_LOAD_IDX_BITS<RegisterClass data_rc>
- : VPseudoInstSI<(outs data_rc:$data),
- (ins SReg_32:$idx, i32imm:$offset, i32imm:$bitsize,
- VSrc_b32:$bitoffset, i32imm:$is_sext_load)> {
+// The lowering normalizes the loaded value to a dword, so only a 32-bit $data
+// form is needed.
+def V_LOAD_IDX_BITS : VPseudoInstSI<(outs VGPR_32:$data),
+ (ins SReg_32:$idx, i32imm:$offset,
+ i32imm:$bitsize, VSrc_b32:$bitoffset,
+ i32imm:$is_sext_load)> {
let mayLoad = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1181,15 +1183,13 @@ class V_LOAD_IDX_BITS<RegisterClass data_rc>
let Defs = [M0];
}
-def V_LOAD_IDX_BITS : V_LOAD_IDX_BITS<VGPR_32>;
-
// Used for byte or short stores.
//
// $data holds the pre-shifted insert value and $mask holds the insert mask
// that is used when lowering to V_BFI_B32.
-class V_STORE_IDX_BITS<RegisterClass data_op_rc>
- : VPseudoInstSI<(outs), (ins data_op_rc:$data, SReg_32:$idx,
- i32imm:$offset, VSrc_b32:$mask)> {
+def V_STORE_IDX_BITS : VPseudoInstSI<(outs),
+ (ins VGPR_32:$data, SReg_32:$idx,
+ i32imm:$offset, VSrc_b32:$mask)> {
let mayStore = 1;
let VALU = 1;
let UseNamedOperandTable = 1;
@@ -1197,32 +1197,25 @@ class V_STORE_IDX_BITS<RegisterClass data_op_rc>
let Defs = [M0];
}
-def V_STORE_IDX_BITS : V_STORE_IDX_BITS<VGPR_32>;
-
// Select the REG_LOAD_BITS/REG_STORE_BITS target nodes into the sub-dword
// indexed pseudos. As for the whole-dword patterns above, an (add idx, imm)
// shape folds a constant dword offset into the pseudo's $offset operand.
-multiclass VRegIdxBitsLoadStorePat<Instruction load_inst,
- Instruction store_inst, ValueType vt> {
- def : GCNPat<
- (vt (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
- (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, imm:$offset, imm:$bitsize, $bitoffset,
- imm:$is_sext_load)>;
- def : GCNPat<
- (vt (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
- (i32 imm:$is_sext_load))),
- (load_inst $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
- (store_inst $data, $idx, imm:$offset, $mask)>;
- def : GCNPat<
- (SIreg_store_bits vt:$data, i32:$idx, i32:$mask),
- (store_inst $data, $idx, 0, $mask)>;
-}
-
-defm : VRegIdxBitsLoadStorePat<V_LOAD_IDX_BITS, V_STORE_IDX_BITS, i32>;
+def : GCNPat<
+ (i32 (SIreg_load_bits (add i32:$idx, (i32 imm:$offset)),
+ (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, imm:$offset, imm:$bitsize, $bitoffset,
+ imm:$is_sext_load)>;
+def : GCNPat<
+ (i32 (SIreg_load_bits i32:$idx, (i32 imm:$bitsize), i32:$bitoffset,
+ (i32 imm:$is_sext_load))),
+ (V_LOAD_IDX_BITS $idx, 0, imm:$bitsize, $bitoffset, imm:$is_sext_load)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, (add i32:$idx, (i32 imm:$offset)), i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, imm:$offset, $mask)>;
+def : GCNPat<
+ (SIreg_store_bits i32:$data, i32:$idx, i32:$mask),
+ (V_STORE_IDX_BITS $data, $idx, 0, $mask)>;
// This is a pseudo variant of the v_movreld_b32 instruction in which the
// vector operand appears only twice, once as def and once as use. Using this
>From 92d7c5d30415b15ad0abf2347aec0dd9c203c93c Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 15:24:43 +0300
Subject: [PATCH 08/15] Cover the remaining VGPR-memory operand invariants
---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 38 +++++++++++++------
1 file changed, 26 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
index bec5ef54aa203..db32b6957abb3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
@@ -1,26 +1,40 @@
# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-# On a movrel subtarget the dword index of a VGPR "as memory" (address space 13)
-# access ends up in M0, which the v_movrel[sd] emitted by
-# AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has rewritten the
-# index to M0, the access must record that the eventual move clobbers M0 -
-# otherwise it could be separated from the code that sets M0 up. Check that the
-# machine verifier rejects a register index without that implicit-def, for both
-# the whole-dword and the sub-dword pseudos.
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
+# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
+# rewritten the index to M0, the access must record that the eventual move
+# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
+# The lowering also relies on the access having exactly one memory operand (it
+# synthesizes a load operand from it for a sub-dword store), and on the index
+# carrying no subregister, since it is rewritten in place to M0.
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
---
-name: missing_m0_implicit_def
+name: operand_invariants
tracksRegLiveness: true
body: |
bb.0:
- liveins: $sgpr0, $vgpr0
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
- %2:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
+
+ ; Missing implicit-def $m0, for each of the four pseudo forms.
+ %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %3:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
+ %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %2, implicit %3
+ S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
...
>From 7638a2481e49be6c30edc84454891ffcb1c08b02 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:07:38 +0300
Subject: [PATCH 09/15] Also build the sub-dword VGPR-memory tests for wave64
---
llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 66790a4a155fe..992b737009947 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,6 +2,14 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
+; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
+; sub-dword code is the same there modulo register allocation, so only compile
+; and verify rather than checking the output a second time. What differs is the
+; waterfall around a divergent access, which uses a 64-bit exec mask.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
; move and extracts the accessed bits with v_bfe_{u,i}32; a store is a
>From 6d1fa3a13ca57cfdeb4716419bb2652ba78c935e Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:28:47 +0300
Subject: [PATCH 10/15] Test sub-dword VGPR-memory accesses across control flow
---
.../as-vgpr-bits-control-flow.ll | 316 ++++++++++++++++++
1 file changed, 316 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
new file mode 100644
index 0000000000000..73edbbb7855e5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+
+; Also compile for a wave64 movrel subtarget; see as-vgpr-bits.ll.
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+
+; Sub-dword (8/16-bit) accesses of the VGPR "as memory" address space (13)
+; across control flow. A sub-dword store is a read-modify-write of the dword
+; containing it, which makes it sensitive to how it is placed relative to other
+; accesses of the same dword, to loop bodies, and to the exec mask.
+
+; Two stores into different bytes of the same dword. Each is a full
+; read-modify-write, so the second has to observe the first: if the two dword
+; loads were merged, the first byte would be lost.
+define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
+; GFX12-SDAG-LABEL: two_bytes_same_dword:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_lshr_b32 s1, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_lshl_b32 s3, s0, 3
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
+; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: two_bytes_same_dword:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v2
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s2, s1, 31
+; GFX12-GISEL-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s1, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 1
+ store i8 %a, ptr addrspace(13) %p
+ store i8 %b, ptr addrspace(13) %q
+ ret void
+}
+
+; A store inside a loop. The pointer deliberately depends on the induction
+; variable: with a loop-invariant pointer the whole access is hoisted out and
+; nothing inside the loop is exercised. M0 has to be re-established for each
+; access on every iteration.
+define void @store_in_loop(ptr addrspace(13) inreg %p, i8 %v, i32 inreg %n) {
+; GFX12-SDAG-LABEL: store_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: .LBB1_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s2, v0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: .LBB1_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s3, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, s4, v0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfi_b32 v1, s2, v1, v2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ store i8 %v, ptr addrspace(13) %q
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Likewise for a load, accumulating so the result cannot be folded away.
+define i32 @load_in_loop(ptr addrspace(13) inreg %p, i32 inreg %n) {
+; GFX12-SDAG-LABEL: load_in_loop:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-SDAG-NEXT: .LBB2_1: ; %loop
+; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-SDAG-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-SDAG-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-SDAG-NEXT: ; %bb.2: ; %exit
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: load_in_loop:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: .LBB2_1: ; %loop
+; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_add_co_i32 s1, s1, -1
+; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, 1
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_cmp_lg_u32 s1, 0
+; GFX12-GISEL-NEXT: v_bfe_u32 v1, v1, s2, 8
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_cbranch_scc1 .LBB2_1
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %q = getelementptr i8, ptr addrspace(13) %p, i32 %i
+ %x = load i8, ptr addrspace(13) %q
+ %z = zext i8 %x to i32
+ %acc.next = add i32 %acc, %z
+ %i.next = add i32 %i, 1
+ %done = icmp eq i32 %i.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; A store under divergent control flow: the read-modify-write must stay inside
+; the conditional region so that lanes which did not take the branch keep their
+; dword unmodified.
+define void @store_divergent_if(ptr addrspace(13) inreg %p, i8 %v, i32 %tid) {
+; GFX12-SDAG-LABEL: store_divergent_if:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-SDAG-NEXT: s_cbranch_execz .LBB3_2
+; GFX12-SDAG-NEXT: ; %bb.1: ; %then
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: .LBB3_2: ; %exit
+; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: store_divergent_if:
+; GFX12-GISEL: ; %bb.0: ; %entry
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-GISEL-NEXT: v_cmpx_gt_u32_e32 8, v1
+; GFX12-GISEL-NEXT: ; %bb.1: ; %then
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s0, 3
+; GFX12-GISEL-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 31
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, 0xff, s2
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_bfi_b32 v0, s2, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX12-GISEL-NEXT: ; %bb.2: ; %exit
+; GFX12-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %c = icmp ult i32 %tid, 8
+ br i1 %c, label %then, label %exit
+
+then:
+ store i8 %v, ptr addrspace(13) %p
+ br label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
>From 20689b60ca2192b9619d520ed300b50f43114b71 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:58:56 +0300
Subject: [PATCH 11/15] Pin VGPR-memory indexed accesses to EXEC and mark them
divergent
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c7c40af261b76..3d659db618cfb 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -189,6 +189,13 @@ bool SIInstrInfo::isReMaterializableImpl(
bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
assert(isVALU(MI, /*AllowLDSDMA=*/true));
+ // A VGPR "as memory" indexed access reads or writes the per-lane vector
+ // registers of the active lanes, so which lanes are active is part of what it
+ // does. Its implicit use of EXEC must not be treated as ignorable, or the
+ // access could be moved across a write to EXEC.
+ if (isa<AMDGPUMI::VLoadStoreIdxInst>(MI))
+ return true;
+
// If it is convergent it depends on EXEC.
if (MI.isConvergent())
return true;
@@ -11155,6 +11162,12 @@ ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
return ValueUniformity::Default;
}
+ // As above for the generic opcodes, but after instruction selection: an
+ // indexed load reads the wave's per-lane view of its vector registers, so
+ // even a uniform index yields a divergent value.
+ if (isa<AMDGPUMI::VLoadIdxInst>(MI))
+ return ValueUniformity::NeverUniform;
+
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
>From 8e329fcf6b7833ee72b79c006d5a6306873c09b2 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 16:59:30 +0300
Subject: [PATCH 12/15] NFC: track the downstream implementation more closely
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 35 ++++++++++++--------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
2 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index e7d3a1e8bf172..815605099cd90 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -36,9 +36,9 @@ using namespace llvm;
namespace {
-class LowerIdxOps {
+class AMDGPULowerIdxOpsImpl {
public:
- LowerIdxOps(MachineFunction &MF)
+ AMDGPULowerIdxOpsImpl(MachineFunction &MF)
: TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()),
MRI(&MF.getRegInfo()) {}
@@ -52,7 +52,7 @@ class LowerIdxOps {
MachineRegisterInfo *MRI;
};
-void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
auto &LoadIdx = cast<AMDGPUMI::VLoadIdxInst>(MI);
@@ -69,19 +69,21 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
SrcAReg)
.add(LoadIdx.getIdxOp())
.add(LoadIdx.getOffsetOp());
- LoadMIB.addMemOperand(*MI.memoperands_begin());
+ auto *LoadMMO = *MI.memoperands_begin();
+ LoadMIB.addMemOperand(LoadMMO);
// Match what instruction selection does for a whole-dword access with a
// register index: record that the M0 write implied by the eventual movrel
// clobbers M0 (see AMDGPUAssignIdxToM0).
if (LoadIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
+ Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
MachineOperand BitOffset = MI.getOperand(4);
if (BitOffset.isImm())
BitOffset.setImm(BitOffset.getImm() & 31);
- BuildMI(*MBB, MI, MI.getDebugLoc(), II, LoadIdx.getDataOp().getReg())
+ BuildMI(*MBB, MI, MI.getDebugLoc(), II, DataReg)
.addReg(SrcAReg)
.add(BitOffset)
.add(MI.getOperand(3)); // bitsize
@@ -91,11 +93,15 @@ void LowerIdxOps::lowerLoadIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
+void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent();
auto &StoreIdx = cast<AMDGPUMI::VStoreIdxInst>(MI);
+ const MCInstrDesc &II = TII->get(AMDGPU::V_BFI_B32_e64);
+
+ Register DataReg = StoreIdx.getDataOp().getReg();
+
Register SrcAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DstAReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
@@ -110,15 +116,16 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
// Synthesize a load MMO from the store's.
auto NewFlags = MachineMemOperand::MOLoad;
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
- LoadMIB.addMemOperand(MF->getMachineMemOperand(StoreMMO, NewFlags));
+ MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
+ LoadMIB.addMemOperand(LoadMMO);
if (StoreIdx.getIdxOp().isReg())
LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
- BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(AMDGPU::V_BFI_B32_e64), DstAReg)
- .add(StoreIdx.getOperand(3)) // mask
- .addReg(StoreIdx.getDataOp().getReg())
- .addReg(SrcAReg);
+ auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
+ CoreMIB.add(StoreIdx.getOperand(3)); // mask
+ CoreMIB.addReg(DataReg);
+ CoreMIB.addReg(SrcAReg);
// Write the dword back.
auto StoreMIB =
@@ -135,7 +142,7 @@ void LowerIdxOps::lowerStoreIdxBits(MachineInstr &MI) {
MI.eraseFromParent();
}
-bool LowerIdxOps::run(MachineFunction &MF) {
+bool AMDGPULowerIdxOpsImpl::run(MachineFunction &MF) {
bool Changed = false;
LLVM_DEBUG(dbgs() << "\nLowerIdxOps on function: " << MF.getName() << "\n");
@@ -171,7 +178,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
// This is required lowering, not an optimization: nothing else expands the
// sub-dword pseudos, and AMDGPULowerVGPREncoding cannot lower them. It
// therefore must not be skipped for optnone functions.
- return LowerIdxOps(MF).run(MF);
+ return AMDGPULowerIdxOpsImpl(MF).run(MF);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -187,7 +194,7 @@ class AMDGPULowerIdxOpsLegacy : public MachineFunctionPass {
PreservedAnalyses
AMDGPULowerIdxOpsPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!LowerIdxOps(MF).run(MF))
+ if (!AMDGPULowerIdxOpsImpl(MF).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b898b37440b85..cb0d7f756fac5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -19002,7 +19002,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
APInt IndexMask =
APInt::getLowBitsSet(32, Log2_32_Ceil(NumAddressableVGPRs));
- unsigned IndexOpIdx = 0, BitOffOpIdx = 0;
+ unsigned IndexOpIdx, BitOffOpIdx = 0;
switch (N->getOpcode()) {
case AMDGPUISD::REG_LOAD:
IndexOpIdx = 1;
>From 8a7e7f8df8316e57302e3fc9f5e06f0660dc7971 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:16:13 +0300
Subject: [PATCH 13/15] Do not describe sub-dword VGPR-memory accesses by index
and width
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3d659db618cfb..e14de0e34fb49 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -551,6 +551,13 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
}
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
+ // A sub-dword access is not described by a dword index and width alone -
+ // the bit position within the dword is part of the address - and has no
+ // entry in the width table. Report it as opaque so that callers such as
+ // areMemAccessesTriviallyDisjoint() fall back to assuming an overlap.
+ if (!AMDGPU::getVLdStIdxOpcodeInfoByOpcode(LdSt.getOpcode()))
+ return false;
+
BaseOp = &LdStIdx->getIdxOp();
OffsetOp = &LdStIdx->getOffsetOp();
>From 704e35c2ab00ecf9c83bcb9bd63fa0ab7f3f66a1 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 17:47:57 +0300
Subject: [PATCH 14/15] Fix tests
---
.../AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll | 15 ++-
.../as-vgpr-index-demanded-bits.ll | 105 ++++++++++++++++++
2 files changed, 115 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
index 992b737009947..4910e9c56c4b7 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits.ll
@@ -2,13 +2,18 @@
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
-; Also compile for a wave64 movrel subtarget. The expansion itself is subtarget
-; independent - AMDGPULowerIdxOps consults no subtarget predicate - and the
-; sub-dword code is the same there modulo register allocation, so only compile
-; and verify rather than checking the output a second time. What differs is the
-; waterfall around a divergent access, which uses a 64-bit exec mask.
+; Also compile for two further movrel subtargets. The expansion itself is
+; subtarget independent - AMDGPULowerIdxOps consults no subtarget predicate - and
+; the sub-dword code is the same on both modulo register allocation, so only
+; compile and verify rather than checking the output again. gfx942 differs in
+; using a 64-bit exec mask for the waterfall around a divergent access, and
+; gfx1250 in having more than 256 addressable VGPRs. The effects of the latter on
+; the dword index are checked in as-vgpr-index-demanded-bits.ll, whose entry
+; functions also cover the S_SET_VGPR_MSB encoding.
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -filetype=null %s
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -filetype=null %s
; End-to-end lowering of sub-dword (8/16-bit) accesses of the VGPR "as memory"
; address space (13). A load reads the containing dword with an M0-relative
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
index 5ce932cb75200..e3ea075016159 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-index-demanded-bits.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-SDAG
+; RUN: llc -global-isel=1 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-GISEL
; The VGPR "as memory" (address space 13) dword index only needs enough bits to
; address all addressable VGPRs, so a redundant high-bit mask feeding the index
@@ -29,6 +31,28 @@ define amdgpu_ps i32 @load_masked_index(i32 inreg %arg) {
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
%v = load i32, ptr addrspace(13) %ptr
@@ -47,6 +71,24 @@ define amdgpu_ps void @store_masked_index(i32 inreg %arg, i32 %val) {
; GFX12-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-LABEL: store_masked_index:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: store_masked_index:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_bfe_u32 m0, s0, 0xe0002
+; GFX1250-GISEL-NEXT: v_movreld_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_endpgm
%idx = and i32 %arg, 65535
%ptr = inttoptr i32 %idx to ptr addrspace(13)
store i32 %val, ptr addrspace(13) %ptr
@@ -82,6 +124,37 @@ define amdgpu_ps i32 @load_bits_masked_bit_offset(i32 inreg %arg, i32 inreg %n)
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_masked_bit_offset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_bfe_u32 m0, s0, 0x80002
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_masked_bit_offset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_and_b32 s0, s0, 0x3ff
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%m = and i32 %sum, 1023
%ptr = inttoptr i32 %m to ptr addrspace(13)
@@ -119,6 +192,37 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-SDAG-LABEL: load_bits_or_high_bit:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-SDAG-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: load_bits_or_high_bit:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_add_co_i32 s0, s0, s1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_bitset1_b32 s0, 16
+; GFX1250-GISEL-NEXT: s_lshr_b32 m0, s0, 2
+; GFX1250-GISEL-NEXT: s_lshl_b32 s0, s0, 3
+; GFX1250-GISEL-NEXT: v_movrels_b32_e32 v0, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, s0, 8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%sum = add i32 %arg, %n
%o = or i32 %sum, 65536
%ptr = inttoptr i32 %o to ptr addrspace(13)
@@ -128,3 +232,4 @@ define amdgpu_ps i32 @load_bits_or_high_bit(i32 inreg %arg, i32 inreg %n) {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12: {{.*}}
+; GFX1250: {{.*}}
>From f73e165183af81ee1d2cc49f6d8f9c543f75f94e Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 29 Jul 2026 18:14:45 +0300
Subject: [PATCH 15/15] Check the encodings of the sub-dword VGPR-memory
accesses
---
llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp | 12 +---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 26 +++----
.../as-vgpr-bits-control-flow.ll | 7 +-
.../AddressSpaceVGPR/as-vgpr-bits-encoding.ll | 71 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-divergent.ll | 15 ++--
.../AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll | 28 ++++----
.../as-vgpr-lower-idx-ops.mir | 16 ++---
.../AddressSpaceVGPR/as-vgpr-verifier-m0.mir | 40 -----------
.../AddressSpaceVGPR/as-vgpr-verifier.mir | 32 +++++++++
9 files changed, 146 insertions(+), 101 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
index 815605099cd90..ec1463e0b2eb2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIdxOps.cpp
@@ -15,7 +15,8 @@
///
/// A sub-dword store is therefore a read-modify-write of the containing dword.
/// This runs before AMDGPUAssignIdxToM0, so the whole-dword accesses created
-/// here take part in the usual M0 setup, and before register allocation
+/// here take part in the usual M0 setup - they declare their write of M0
+/// themselves, like the pseudo they replace - and before register allocation
/// because it introduces new virtual registers.
//
//===----------------------------------------------------------------------===//
@@ -71,11 +72,6 @@ void AMDGPULowerIdxOpsImpl::lowerLoadIdxBits(MachineInstr &MI) {
.add(LoadIdx.getOffsetOp());
auto *LoadMMO = *MI.memoperands_begin();
LoadMIB.addMemOperand(LoadMMO);
- // Match what instruction selection does for a whole-dword access with a
- // register index: record that the M0 write implied by the eventual movrel
- // clobbers M0 (see AMDGPUAssignIdxToM0).
- if (LoadIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
Register DataReg = LoadIdx.getDataOp().getReg();
// Extract the accessed bits out of it.
@@ -118,8 +114,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
NewFlags |= StoreMMO->getFlags() & ~MachineMemOperand::MOStore;
MachineMemOperand *LoadMMO = MF->getMachineMemOperand(StoreMMO, NewFlags);
LoadMIB.addMemOperand(LoadMMO);
- if (StoreIdx.getIdxOp().isReg())
- LoadMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
// Insert the stored bits into it.
auto CoreMIB = BuildMI(*MBB, MI, MI.getDebugLoc(), II, DstAReg);
@@ -134,8 +128,6 @@ void AMDGPULowerIdxOpsImpl::lowerStoreIdxBits(MachineInstr &MI) {
.add(StoreIdx.getIdxOp())
.add(StoreIdx.getOffsetOp());
StoreMIB.addMemOperand(StoreMMO);
- if (StoreIdx.getIdxOp().isReg())
- StoreMIB.addReg(AMDGPU::M0, RegState::ImplicitDefine);
LLVM_DEBUG(dbgs() << " *** Expanded pseudo: "; MI.print(dbgs()));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e14de0e34fb49..03d430d5b6992 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5232,11 +5232,11 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
return false;
}
- // A VGPR "as memory" indexed access carries its dword index in an SGPR, which
- // AMDGPUAssignIdxToM0 rewrites to M0 - the register read by the v_movrel[sd]
- // that AMDGPULowerVGPREncoding eventually emits. Until that rewrite the
- // access has to record that the move clobbers M0, otherwise it could be
- // separated from the code that sets M0 up.
+ // The lowering of a VGPR "as memory" indexed access relies on it having
+ // exactly one memory operand, since a sub-dword store synthesizes a load
+ // operand from it, and on the index carrying no subregister, since
+ // AMDGPUAssignIdxToM0 rewrites it in place to M0. That the access writes M0 is
+ // declared by the instruction itself, so it needs no check here.
if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
if (MI.getNumMemOperands() != 1) {
ErrInfo = "v_load/store_idx should have exactly one memory operand.";
@@ -5244,19 +5244,9 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
const MachineOperand &IdxOp = LdStIdx->getIdxOp();
- if (IdxOp.isReg()) {
- if (IdxOp.getSubReg() != 0) {
- ErrInfo =
- "v_load/store_idx register index must not have a subregister.";
- return false;
- }
-
- if (ST.hasMovrel() && IdxOp.getReg() != AMDGPU::M0 &&
- !MI.definesRegister(AMDGPU::M0, /*TRI=*/nullptr)) {
- ErrInfo = "v_load/store_idx with register index must have implicit-def "
- "$m0 on movrel subtargets.";
- return false;
- }
+ if (IdxOp.isReg() && IdxOp.getSubReg() != 0) {
+ ErrInfo = "v_load/store_idx register index must not have a subregister.";
+ return false;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
index 73edbbb7855e5..1f20bf28b2335 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-control-flow.ll
@@ -37,15 +37,16 @@ define void @two_bytes_same_dword(ptr addrspace(13) inreg %p, i8 %a, i8 %b) {
; GFX12-SDAG-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX12-SDAG-NEXT: s_lshr_b32 s0, s0, 2
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v1, s3, v1
; GFX12-SDAG-NEXT: s_lshl_b32 s1, 0xff, s3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
-; GFX12-SDAG-NEXT: v_movrels_b32_e32 v0, v0
+; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v0, s3, v1
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-SDAG-NEXT: s_mov_b32 m0, s0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v1, v0
+; GFX12-SDAG-NEXT: v_bfi_b32 v0, s1, v0, v1
; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
new file mode 100644
index 0000000000000..32921a6244ebe
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-bits-encoding.ll
@@ -0,0 +1,71 @@
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1200 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1200 - | FileCheck --check-prefix=DIS %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx1250 -o - %s | llvm-mc -triple=amdgcn -mcpu=gfx1250 -filetype=obj -o - | llvm-objdump -d --mcpu=gfx1250 - | FileCheck --check-prefixes=DIS,GFX1250 %s
+
+; The sub-dword accesses of the VGPR "as memory" address space (13) that
+; AMDGPULowerIdxOps expands, taken to machine code and back. The other tests here
+; check assembly text, which does not show that the operand forms the expansion
+; picks can be assembled and encoded: a mask it builds from a known bit offset is
+; too wide for an inline constant and becomes a 32-bit literal, and a bit offset
+; reaches the bit-field extract as either an inline constant or an SGPR. On a
+; subtarget with more than 256 addressable VGPRs the whole-dword accesses the
+; expansion creates additionally carry the S_SET_VGPR_MSB encoding, so check
+; there too.
+;
+; The expansion runs after instruction selection and is handed the same pseudos
+; by both selectors, which produce identical code for these functions, so one
+; selector is enough here. as-vgpr-bits.ll covers the two separately.
+
+; A statically known bit offset turns the mask into a literal operand of the
+; bit-field insert.
+define amdgpu_ps void @store_i8_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_aligned>:
+; GFX1250: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+define amdgpu_ps void @store_i16_aligned(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i16_aligned>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, 0xffff, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i16
+ store i16 %v, ptr addrspace(13) %p, align 4
+ ret void
+}
+
+; Without one the mask is computed at run time and is an SGPR instead.
+define amdgpu_ps void @store_i8_dynamic_mask(ptr addrspace(13) inreg %p, i32 inreg %vv) {
+; DIS-LABEL: <store_i8_dynamic_mask>:
+; DIS: v_movrels_b32_e32 v1, v0
+; DIS: v_bfi_b32 v0, s1, v0, v1
+; DIS: v_movreld_b32_e32 v0, v0
+ %v = trunc i32 %vv to i8
+ store i8 %v, ptr addrspace(13) %p
+ ret void
+}
+
+; Both bit offset and width of the extract are inline constants here.
+define amdgpu_ps i32 @load_i8_zext_aligned(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_zext_aligned>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_u32 v0, v0, 0, 8
+ %x = load i8, ptr addrspace(13) %p, align 4
+ %z = zext i8 %x to i32
+ ret i32 %z
+}
+
+; A run-time bit offset is an SGPR, and the sign-extending load selects the
+; signed extract.
+define amdgpu_ps i32 @load_i8_sext_dynamic(ptr addrspace(13) inreg %p) {
+; DIS-LABEL: <load_i8_sext_dynamic>:
+; DIS: v_movrels_b32_e32 v0, v0
+; DIS: v_bfe_i32 v0, v0, s0, 8
+ %x = load i8, ptr addrspace(13) %p
+ %s = sext i8 %x to i32
+ ret i32 %s
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
index 0cde03f7ab74c..28aa0ff7f0fb9 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-divergent.ll
@@ -47,20 +47,19 @@ define i32 @load_i32(ptr addrspace(13) %p) {
; GFX12-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX12-GISEL-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX12-GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 s1, exec_lo
; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s1, v0
+; GFX12-GISEL-NEXT: v_cmpx_eq_u32_e32 s2, v0
+; GFX12-GISEL-NEXT: s_mov_b32 m0, s2
; GFX12-GISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v1, v0
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
+; GFX12-GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX12-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-GISEL-NEXT: ; %bb.2:
-; GFX12-GISEL-NEXT: s_mov_b32 m0, s1
-; GFX12-GISEL-NEXT: v_movrels_b32_e32 v0, v0
; GFX12-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
%x = load i32, ptr addrspace(13) %p
%y = add i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
index 1d3de91fd8bc2..cebc6ea23c83b 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-gpr-idx-mode.ll
@@ -60,40 +60,40 @@ define i32 @load_i32_divergent(ptr addrspace(13) %p) {
; GFX9-SDAG-LABEL: load_i32_divergent:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], exec
; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-SDAG-NEXT: s_nop 1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
+; GFX9-SDAG-NEXT: s_set_gpr_idx_off
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-SDAG-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v0
-; GFX9-SDAG-NEXT: s_set_gpr_idx_off
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: load_i32_divergent:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 2, v0
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 2, v0
; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], exec
; GFX9-GISEL-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-GISEL-NEXT: s_nop 1
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, s4, v1
; GFX9-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9-GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
-; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
-; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_set_gpr_idx_on s4, gpr_idx(SRC0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v0
; GFX9-GISEL-NEXT: s_set_gpr_idx_off
+; GFX9-GISEL-NEXT: ; implicit-def: $vgpr1
+; GFX9-GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-GISEL-NEXT: ; %bb.2:
; GFX9-GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%v = load i32, ptr addrspace(13) %p, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
index c991990db933e..c8b4fefa3c3e3 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-lower-idx-ops.mir
@@ -21,7 +21,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -39,7 +39,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[V_LOAD_IDX_B32_]], 0, 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_I32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -57,7 +57,7 @@ body: |
; CHECK: liveins: $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 3, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], 16, 16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -78,7 +78,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_LOAD_IDX_B32_]], [[COPY1]], 8, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_BFE_U32_e64_]]
%0:sreg_32 = COPY $sgpr0
@@ -100,9 +100,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit $exec, implicit-def $m0 :: (load (s8), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 0, implicit-def $m0, implicit $exec :: (load (s8), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 255, [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit $exec, implicit-def $m0 :: (store (s8), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], killed [[COPY]], 0, implicit-def $m0, implicit $exec :: (store (s8), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
@@ -122,9 +122,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit $exec, implicit-def $m0 :: (load (s16), addrspace 13)
+ ; CHECK-NEXT: [[V_LOAD_IDX_B32_:%[0-9]+]]:vgpr_32 = V_LOAD_IDX_B32 [[COPY]], 2, implicit-def $m0, implicit $exec :: (load (s16), addrspace 13)
; CHECK-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 [[COPY2]], [[COPY1]], [[V_LOAD_IDX_B32_]], implicit $exec
- ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit $exec, implicit-def $m0 :: (store (s16), addrspace 13)
+ ; CHECK-NEXT: V_STORE_IDX_B32 [[V_BFI_B32_e64_]], [[COPY]], 2, implicit-def $m0, implicit $exec :: (store (s16), addrspace 13)
; CHECK-NEXT: S_ENDPGM 0
%0:sreg_32 = COPY $sgpr0
%1:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
deleted file mode 100644
index db32b6957abb3..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier-m0.mir
+++ /dev/null
@@ -1,40 +0,0 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
-
-# Operand invariants of the VGPR "as memory" (address space 13) indexed
-# accesses, as enforced by SIInstrInfo::verifyInstruction.
-#
-# On a movrel subtarget the dword index ends up in M0, which the v_movrel[sd]
-# emitted by AMDGPULowerVGPREncoding reads. Until AMDGPUAssignIdxToM0 has
-# rewritten the index to M0, the access must record that the eventual move
-# clobbers M0 - otherwise it could be separated from the code that sets M0 up.
-# The lowering also relies on the access having exactly one memory operand (it
-# synthesizes a load operand from it for a sub-dword store), and on the index
-# carrying no subregister, since it is rewritten in place to M0.
-
-# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
-# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
-# CHECK-COUNT-4: *** Bad machine code: v_load/store_idx with register index must have implicit-def $m0 on movrel subtargets. ***
-
----
-name: operand_invariants
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
- %0:sreg_32 = COPY $sgpr0
- %1:vgpr_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr2_sgpr3
-
- ; No memory operand.
- %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec, implicit-def $m0
-
- ; Index with a subregister.
- %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit $exec, implicit-def $m0 :: (load (s32), addrspace 13)
-
- ; Missing implicit-def $m0, for each of the four pseudo forms.
- %5:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit $exec :: (load (s32), addrspace 13)
- V_STORE_IDX_B32 %1, %0, 0, implicit $exec :: (store (s32), addrspace 13)
- %6:vgpr_32 = V_LOAD_IDX_BITS %0, 0, 8, 0, 0, implicit $exec :: (load (s8), addrspace 13)
- V_STORE_IDX_BITS %1, %0, 0, 255, implicit $exec :: (store (s8), addrspace 13)
- S_ENDPGM 0, implicit %3, implicit %4, implicit %5, implicit %6
-...
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
new file mode 100644
index 0000000000000..d4e7ec49e1f40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-verifier.mir
@@ -0,0 +1,32 @@
+# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=none -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck %s
+
+# Operand invariants of the VGPR "as memory" (address space 13) indexed
+# accesses, as enforced by SIInstrInfo::verifyInstruction.
+#
+# The lowering relies on the access having exactly one memory operand, since a
+# sub-dword store synthesizes a load operand from it, and on the index carrying
+# no subregister, since AMDGPUAssignIdxToM0 rewrites it in place to M0. That the
+# access writes M0 is declared by the instructions themselves, so it needs no
+# check: MIR without the operand does not parse.
+
+# CHECK: *** Bad machine code: v_load/store_idx should have exactly one memory operand. ***
+# CHECK: *** Bad machine code: v_load/store_idx register index must not have a subregister. ***
+
+---
+name: operand_invariants
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr2_sgpr3, $vgpr0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+
+ ; No memory operand.
+ %3:vgpr_32 = V_LOAD_IDX_B32 %0, 0, implicit-def $m0, implicit $exec
+
+ ; Index with a subregister.
+ %4:vgpr_32 = V_LOAD_IDX_B32 %2.sub0, 0, implicit-def $m0, implicit $exec :: (load (s32), addrspace 13)
+
+ S_ENDPGM 0, implicit %3, implicit %4
+...
More information about the llvm-branch-commits
mailing list