[llvm] [AMDGPU] MC support for indexed VBUFFER resources for gfx13 (PR #218649)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 02:46:52 PDT 2026
https://github.com/sstipano created https://github.com/llvm/llvm-project/pull/218649
None
>From e042e13765a1cdd438ff05305217aabbff489a6b Mon Sep 17 00:00:00 2001
From: sstipano <sstipano at amd.com>
Date: Mon, 22 Jun 2026 14:09:23 +0200
Subject: [PATCH] [AMDGPU] MC support for indexed VBUFFER resources for gfx13
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 5 +
.../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 32 ++
llvm/lib/Target/AMDGPU/BUFInstructions.td | 274 +++++++++++-------
.../Disassembler/AMDGPUDisassembler.cpp | 30 +-
.../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp | 12 +
.../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h | 2 +
.../MCTargetDesc/AMDGPUMCCodeEmitter.cpp | 20 ++
llvm/lib/Target/AMDGPU/SIRegisterInfo.td | 26 ++
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 +
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 4 +
llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mtbuf.s | 9 +
llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf.s | 64 ++++
.../MC/AMDGPU/gfx13_asm_vbuffer_mubuf_err.s | 16 +
.../Inputs/amdgpu_isel.ll.expected | 4 +-
14 files changed, 388 insertions(+), 114 deletions(-)
create mode 100644 llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf_err.s
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 160e91cb0da66..7f99406f4f54f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -966,6 +966,10 @@ defm CubeInsts : AMDGPUSubtargetFeature<"cube-insts",
"Has v_cube* instructions"
>;
+defm IndexedResources : AMDGPUSubtargetFeature<"indexed-resources",
+ "Has indexed resources"
+>;
+
defm LerpInst : AMDGPUSubtargetFeature<"lerp-inst",
"Has v_lerp_u8 instruction"
>;
@@ -2531,6 +2535,7 @@ def FeatureISAVersion13 : FeatureSet<
FeatureCvtPkNormVOP3Insts,
FeatureSmemPrefetchInsts,
FeatureAsyncLoadToLDSInsts,
+ FeatureIndexedResources
]>;
def FeatureISAVersion13_Generic: FeatureSet<
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 9ae621fdc6b71..94dfe21a41c1c 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -343,6 +343,8 @@ class AMDGPUOperand : public MCParsedAsmOperand {
bool isVReg32OrOff() const { return isOff() || isVReg32(); }
+ bool isRsrcReg32() const { return isRegClass(AMDGPU::RsrcReg32RegClassID); }
+
bool isNull() const { return isRegKind() && getReg() == AMDGPU::SGPR_NULL; }
bool isAV_LdSt_32_Align2_RegOp() const {
@@ -1706,6 +1708,7 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
bool AllowImm = true);
ParseStatus parseRegWithFPInputMods(OperandVector &Operands);
ParseStatus parseRegWithIntInputMods(OperandVector &Operands);
+ ParseStatus parseRsrcReg(OperandVector &Operands);
ParseStatus parseVReg32OrOff(OperandVector &Operands);
ParseStatus tryParseIndexKey(OperandVector &Operands,
AMDGPUOperand::ImmTy ImmTy);
@@ -3600,6 +3603,35 @@ ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
return parseRegOrImmWithIntInputMods(Operands, false);
}
+ParseStatus AMDGPUAsmParser::parseRsrcReg(OperandVector &Operands) {
+ // Without the marker, fall back to plain register parsing so the legacy
+ // bare-register form (e.g. `s8`, `v8`) still assembles for indexed
+ // buffer/image instructions.
+ if (!trySkipId("rsrcidx"))
+ return parseReg(Operands);
+
+ if (!skipToken(AsmToken::LParen, "expected left paren after rsrcidx"))
+ return ParseStatus::Failure;
+
+ SMLoc RegLoc = getLoc();
+ std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
+ if (!Reg)
+ return ParseStatus::Failure;
+
+ // Enforce that the inner register is a valid index register. The matcher
+ // predicate alone is not sufficient: if it fails, the matcher will fall back
+ // to a non-indexed instruction variant whose resource operand happens to
+ // accept the same register, silently dropping the `rsrcidx` intent.
+ if (!Reg->isRsrcReg32())
+ return Error(RegLoc, "rsrcidx operand must be a 32-bit SGPR or VGPR");
+
+ if (!skipToken(AsmToken::RParen, "expected closing parenthesis"))
+ return ParseStatus::Failure;
+
+ Operands.push_back(std::move(Reg));
+ return ParseStatus::Success;
+}
+
ParseStatus AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
auto Loc = getLoc();
if (trySkipId("off")) {
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 5faa478407c34..2175772b00b89 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -54,6 +54,7 @@ class BUF_Pseudo <string opName, dag outs, dag ins,
InstSI<outs, ins, "", pattern>,
SIMCInstr<opName, SIEncodingFamily.NONE> {
+ bit isIndexed = 0;
let isPseudo = 1;
let isCodeGenOnly = 1;
let Size = 8;
@@ -159,14 +160,15 @@ class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> :
}
class getMTBUFInsDA<list<RegisterOperand> vdataList,
- list<RegisterOperand> vaddrList=[], bit hasRestrictedSOffset> {
+ list<RegisterOperand> vaddrList=[], bit hasRestrictedSOffset,
+ DAGOperand RsrcRC = SReg_128_XNULL> {
RegisterOperand vdata_op = !if(!empty(vdataList), ?, !head(vdataList));
RegisterOperand vaddr_op = !if(!empty(vaddrList), ?, !head(vaddrList));
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset),
(ins SCSrc_b32:$soffset));
- dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset,
+ dag NonVaddrInputs = !con((ins RsrcRC:$srsrc), SOffset,
(ins Offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz));
dag Inputs = !if(!empty(vaddrList),
@@ -177,13 +179,13 @@ class getMTBUFInsDA<list<RegisterOperand> vdataList,
!con((ins vdata_op:$vdata), Inputs));
}
-class getMTBUFIns<int addrKind, list<RegisterOperand> vdataList=[], bit hasRestrictedSOffset> {
+class getMTBUFIns<int addrKind, list<RegisterOperand> vdataList=[], bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
dag ret =
- !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset>.ret,
- !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPROp_32], hasRestrictedSOffset>.ret,
- !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPROp_32], hasRestrictedSOffset>.ret,
- !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VGPROp_64], hasRestrictedSOffset>.ret,
- !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VGPROp_64], hasRestrictedSOffset>.ret,
+ !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPROp_32], hasRestrictedSOffset, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPROp_32], hasRestrictedSOffset, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VGPROp_64], hasRestrictedSOffset, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VGPROp_64], hasRestrictedSOffset, RsrcRC>.ret,
(ins))))));
}
@@ -219,37 +221,40 @@ class MTBUF_Load_Pseudo <string opName,
RegisterOperand vdataClass,
int elems,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[]>
: MTBUF_Pseudo<opName,
(outs vdataClass:$vdata),
- getMTBUFIns<addrKind, [], hasRestrictedSOffset>.ret,
+ getMTBUFIns<addrKind, [], hasRestrictedSOffset, RsrcRC>.ret,
getMTBUFAsmOps<addrKind>.ret,
pattern>,
MTBUF_SetupAddr<addrKind> {
- let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # "_" # getAddrName<addrKind>.ret;
let mayLoad = 1;
let mayStore = 0;
let elements = elems;
}
multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterOperand vdataClass,
- int elems, bit hasRestrictedSOffset> {
+ int elems, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
- def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,
+ def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset, RsrcRC>,
MTBUFAddr64Table<0, NAME>;
- def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,
+ def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset, RsrcRC>,
MTBUFAddr64Table<1, NAME>;
- def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
- def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
- def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
+ def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
let DisableWQM = 1 in {
- def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;
- def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
- def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
- def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
+ def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
}
}
@@ -257,6 +262,7 @@ multiclass MTBUF_Pseudo_Loads<string opName, RegisterOperand vdataClass,
int elems> {
defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>;
defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>;
+ defm _VBUFFER_indexed : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1, RsrcReg128Op>;
}
class MTBUF_Store_Pseudo <string opName,
@@ -264,37 +270,40 @@ class MTBUF_Store_Pseudo <string opName,
RegisterOperand vdataClass,
int elems,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[]>
: MTBUF_Pseudo<opName,
(outs),
- getMTBUFIns<addrKind, [vdataClass], hasRestrictedSOffset>.ret,
+ getMTBUFIns<addrKind, [vdataClass], hasRestrictedSOffset, RsrcRC>.ret,
getMTBUFAsmOps<addrKind>.ret,
pattern>,
MTBUF_SetupAddr<addrKind> {
- let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # "_" # getAddrName<addrKind>.ret;
let mayLoad = 0;
let mayStore = 1;
let elements = elems;
}
multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterOperand vdataClass,
- int elems, bit hasRestrictedSOffset> {
+ int elems, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
- def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>,
+ def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset, RsrcRC>,
MTBUFAddr64Table<0, NAME>;
- def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>,
+ def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset, RsrcRC>,
MTBUFAddr64Table<1, NAME>;
- def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
- def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
- def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
+ def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
let DisableWQM = 1 in {
- def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>;
- def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>;
- def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>;
- def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>;
+ def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset, RsrcRC>;
}
}
@@ -302,6 +311,7 @@ multiclass MTBUF_Pseudo_Stores<string opName, RegisterOperand vdataClass,
int elems> {
defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>;
defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>;
+ defm _VBUFFER_indexed : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1, RsrcReg128Op>;
}
//===----------------------------------------------------------------------===//
@@ -418,14 +428,14 @@ class getBUFVDataRegisterOperandForOp<RegisterOperand Op, bit isTFE> {
class getMUBUFInsDA<list<RegisterOperand> vdataList,
list<RegisterOperand> vaddrList, bit isTFE, bit hasRestrictedSOffset,
- bit isTrue16, bit isLds> {
+ bit isTrue16, bit isLds, DAGOperand RsrcRC = SReg_128_XNULL> {
RegisterOperand vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
RegisterOperand vaddr_op = !if(!empty(vaddrList), ?, !head(vaddrList));
RegisterOperand vdata_op = getBUFVDataRegisterOperand<!cast<SIRegisterClassLike>(vdataClass.RegClass).Size, isTFE, isTrue16>.ret;
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
dag IsAsyncOpnd = !if(isLds, (ins i1imm_0:$IsAsync), (ins));
- dag NonVaddrInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz), IsAsyncOpnd);
+ dag NonVaddrInputs = !con((ins RsrcRC:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz), IsAsyncOpnd);
dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddr_op:$vaddr), NonVaddrInputs));
dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs));
@@ -451,13 +461,13 @@ class getMUBUFElements<ValueType vt> {
}
class getMUBUFIns<int addrKind, list<RegisterOperand> vdataList, bit isTFE,
- bit hasRestrictedSOffset, bit isTrue16, bit isLds> {
+ bit hasRestrictedSOffset, bit isTrue16, bit isLds, DAGOperand RsrcRC = SReg_128_XNULL> {
dag ret =
- !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset, isTrue16, isLds>.ret,
- !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPROp_32], isTFE, hasRestrictedSOffset, isTrue16, isLds>.ret,
- !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPROp_32], isTFE, hasRestrictedSOffset, isTrue16, isLds>.ret,
- !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VGPROp_64], isTFE, hasRestrictedSOffset, isTrue16, isLds>.ret,
- !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VGPROp_64], isTFE, hasRestrictedSOffset, isTrue16, isLds>.ret,
+ !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset, isTrue16, isLds, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPROp_32], isTFE, hasRestrictedSOffset, isTrue16, isLds, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPROp_32], isTFE, hasRestrictedSOffset, isTrue16, isLds, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VGPROp_64], isTFE, hasRestrictedSOffset, isTrue16, isLds, RsrcRC>.ret,
+ !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VGPROp_64], isTFE, hasRestrictedSOffset, isTrue16, isLds, RsrcRC>.ret,
(ins))))));
}
@@ -498,16 +508,19 @@ class MUBUF_Load_Pseudo <string opName,
bit isLdsOpc = 0,
bit isTFE = 0,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[],
RegisterOperand vdata_op = getBUFVDataRegisterOperand<vdata_vt.Size, isTFE>.ret>
: MUBUF_Pseudo<opName,
!if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)),
- !con(getMUBUFIns<addrKind, [], isTFE, hasRestrictedSOffset, 0, isLds>.ret,
+ !con(getMUBUFIns<addrKind, [], isTFE, hasRestrictedSOffset, 0, isLds, RsrcRC>.ret,
!if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))),
getMUBUFAsmOps<addrKind, !or(isLds, isLdsOpc), isLds, isTFE>.ret,
pattern>,
MUBUF_SetupAddr<addrKind> {
- let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") #
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") #
"_" # getAddrName<addrKind>.ret;
let AsmMatchConverter = "cvtMubuf";
@@ -548,50 +561,50 @@ multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPa
}
multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt,
- bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> {
+ bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt);
- def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,
+ def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>,
MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
- def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>,
+ def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>,
MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
- def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
- def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
- def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
+ def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
let DisableWQM = 1 in {
- def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
- def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
- def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
- def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>;
+ def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC>;
}
}
multiclass MUBUF_Pseudo_Loads_Helper_t16<string opName, string Hi16Name, string Lo16Name, ValueType load_vt,
- bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> {
- def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
+ def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>, True16D16Table<Hi16Name#"_OFFSET", Lo16Name#"_OFFSET">;
- def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>, True16D16Table<Hi16Name#"_ADDR64", Lo16Name#"_ADDR64">;
- def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_OFFEN", Lo16Name#"_OFFEN">;
- def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_IDXEN", Lo16Name#"_IDXEN">;
- def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_BOTHEN", Lo16Name#"_BOTHEN">;
let DisableWQM = 1 in {
- def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_OFFSET_exact", Lo16Name#"_OFFSET_exact">;
- def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_OFFEN_exact", Lo16Name#"_OFFEN_exact">;
- def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_IDXEN_exact", Lo16Name#"_IDXEN_exact">;
- def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, [], VGPROp_16>,
+ def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset, RsrcRC, [], VGPROp_16>,
True16D16Table<Hi16Name#"_BOTHEN_exact", Lo16Name#"_BOTHEN_exact">;
}
}
@@ -600,10 +613,12 @@ multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32,
bit TiedDest = 0, bit isLds = 0> {
defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>;
defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>;
+ defm _VBUFFER_indexed : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1, RsrcReg128Op>;
if !not(isLds) then {
defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>;
defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>;
+ defm _TFE_VBUFFER_indexed : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1, RsrcReg128Op>;
}
}
@@ -612,15 +627,18 @@ multiclass MUBUF_Pseudo_Loads_t16<string opName, ValueType load_vt = i32,
let True16Predicate = NotUseRealTrue16Insts in {
defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>;
defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>;
+ defm _VBUFFER_indexed : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1, RsrcReg128Op>;
}
let True16Predicate = UseRealTrue16Insts in {
defvar NAME16 = opName#"_t16";
defm _t16 : MUBUF_Pseudo_Loads_Helper_t16<NAME16, hiOpName, NAME, i16, 0, isLds, 0, 0>;
defm _t16_VBUFFER : MUBUF_Pseudo_Loads_Helper_t16<NAME16, hiOpName#"_VBUFFER", NAME#"_VBUFFER", i16, 0, isLds, 0, 1>;
+ defm _t16_VBUFFER_indexed : MUBUF_Pseudo_Loads_Helper_t16<NAME16, hiOpName#"_VBUFFER_indexed", NAME#"_VBUFFER_indexed", i16, 0, isLds, 0, 1, RsrcReg128Op>;
}
if !not(isLds) then {
defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>;
defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>;
+ defm _TFE_VBUFFER_indexed : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1, RsrcReg128Op>;
}
}
@@ -642,15 +660,18 @@ class MUBUF_Store_Pseudo <string opName,
ValueType store_vt,
bit isTFE = 0,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[],
bit isTrue16 = false>
: MUBUF_Pseudo<opName,
(outs),
- getMUBUFIns<addrKind, [getVregSrcForVT<store_vt, isTrue16, 0>.ret], isTFE, hasRestrictedSOffset, isTrue16, 0>.ret,
+ getMUBUFIns<addrKind, [getVregSrcForVT<store_vt, isTrue16, 0>.ret], isTFE, hasRestrictedSOffset, isTrue16, 0, RsrcRC>.ret,
getMUBUFAsmOps<addrKind, 0, 0, isTFE>.ret,
pattern>,
MUBUF_SetupAddr<addrKind> {
- let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") #
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # "_" # !if(isTFE, "_tfe", "") #
getAddrName<addrKind>.ret;
let mayLoad = 0;
let mayStore = 1;
@@ -677,50 +698,50 @@ multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SD
}
multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt,
- bit isTFE, bit hasRestrictedSOffset> {
+ bit isTFE, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt);
- def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>,
+ def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>,
MUBUFAddr64Table<0, NAME>;
- def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset>,
+ def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>,
MUBUFAddr64Table<1, NAME>;
- def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
- def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
- def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
+ def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
let DisableWQM = 1 in {
- def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>;
- def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
- def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
- def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>;
+ def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
+ def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset, RsrcRC>;
}
}
multiclass MUBUF_Pseudo_Stores_Helper_t16<string opName, string Hi16Name, string Lo16Name, ValueType store_vt,
- bit isTFE, bit hasRestrictedSOffset> {
- def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ bit isTFE, bit hasRestrictedSOffset, DAGOperand RsrcRC = SReg_128_XNULL> {
+ def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
MUBUFAddr64Table<0, NAME>, True16D16Table<Hi16Name#"_OFFSET", Lo16Name#"_OFFSET">;
- def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
MUBUFAddr64Table<1, NAME>, True16D16Table<Hi16Name#"_ADDR64", Lo16Name#"_ADDR64">;
- def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_OFFEN", Lo16Name#"_OFFEN">;
- def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_IDXEN", Lo16Name#"_IDXEN">;
- def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_BOTHEN", Lo16Name#"_BOTHEN">;
let DisableWQM = 1 in {
- def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_OFFSET_exact", Lo16Name#"_OFFSET_exact">;
- def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_OFFEN_exact", Lo16Name#"_OFFEN_exact">;
- def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_IDXEN_exact", Lo16Name#"_IDXEN_exact">;
- def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt, isTFE, hasRestrictedSOffset, [], 1/*isTrue16*/>,
+ def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt, isTFE, hasRestrictedSOffset, RsrcRC, [], 1/*isTrue16*/>,
True16D16Table<Hi16Name#"_BOTHEN_exact", Lo16Name#"_BOTHEN_exact">;
}
}
@@ -730,22 +751,27 @@ multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> {
defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>;
defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>;
+ defm _VBUFFER_indexed : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1, RsrcReg128Op>;
defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>;
+ defm _TFE_VBUFFER_indexed : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1, RsrcReg128Op>;
}
multiclass MUBUF_Pseudo_Stores_t16<string opName, ValueType store_vt = i32> {
defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>;
defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>;
+ defm _TFE_VBUFFER_indexed : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1, RsrcReg128Op>;
let True16Predicate = NotUseRealTrue16Insts in {
defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>;
defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>;
+ defm _VBUFFER_indexed : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1, RsrcReg128Op>;
}
let True16Predicate = UseRealTrue16Insts, SubtargetPredicate = HasD16LoadStore in {
defvar NAME16 = opName#"_t16";
defm _t16 : MUBUF_Pseudo_Stores_Helper_t16<NAME16, NAME#"_D16_HI", NAME, i16, 0, 0>;
defm _t16_VBUFFER : MUBUF_Pseudo_Stores_Helper_t16<NAME16, NAME#"_D16_HI_VBUFFER", NAME#"_VBUFFER", i16, 0, 1>;
+ defm _t16_VBUFFER_indexed : MUBUF_Pseudo_Stores_Helper_t16<NAME16, NAME#"_D16_HI_VBUFFER", NAME#"_VBUFFER", i16, 0, 1, RsrcReg128Op>;
}
}
@@ -768,13 +794,13 @@ class MUBUF_Pseudo_Store_Lds<string opName>
}
class getMUBUFAtomicInsDA<RegisterOperand vdata_op, bit vdata_in, bit hasRestrictedSOffset,
- list<RegisterOperand> vaddrList=[]> {
+ DAGOperand RsrcRC, list<RegisterOperand> vaddrList=[]> {
RegisterOperand vaddr_op = !if(!empty(vaddrList), ?, !head(vaddrList));
dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata));
dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddr_op:$vaddr)));
dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
- dag MainInputs = !con((ins SReg_128_XNULL:$srsrc), SOffset, (ins Offset:$offset));
+ dag MainInputs = !con((ins RsrcRC:$srsrc), SOffset, (ins Offset:$offset));
dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol),
(ins CPol_NonGLC_WithDefault:$cpol));
@@ -784,18 +810,19 @@ class getMUBUFAtomicInsDA<RegisterOperand vdata_op, bit vdata_in, bit hasRestric
class getMUBUFAtomicIns<int addrKind,
RegisterOperand vdataClass,
bit vdata_in,
- bit hasRestrictedSOffset> {
+ bit hasRestrictedSOffset,
+ DAGOperand RsrcRC=SReg_128_XNULL> {
dag ret =
!if(!eq(addrKind, BUFAddrKind.Offset),
- getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset>.ret,
+ getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, RsrcRC>.ret,
!if(!eq(addrKind, BUFAddrKind.OffEn),
- getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPROp_32]>.ret,
+ getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, RsrcRC, [VGPROp_32]>.ret,
!if(!eq(addrKind, BUFAddrKind.IdxEn),
- getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPROp_32]>.ret,
+ getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, RsrcRC, [VGPROp_32]>.ret,
!if(!eq(addrKind, BUFAddrKind.BothEn),
- getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPROp_64]>.ret,
+ getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, RsrcRC, [VGPROp_64]>.ret,
!if(!eq(addrKind, BUFAddrKind.Addr64),
- getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, [VGPROp_64]>.ret,
+ getMUBUFAtomicInsDA<vdataClass, vdata_in, hasRestrictedSOffset, RsrcRC, [VGPROp_64]>.ret,
(ins))))));
}
@@ -820,13 +847,16 @@ class MUBUF_Atomic_Pseudo<string opName,
class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
RegisterOperand vdataClass,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[]>
: MUBUF_Atomic_Pseudo<opName, addrKind,
(outs),
- getMUBUFAtomicIns<addrKind, vdataClass, 0, hasRestrictedSOffset>.ret,
+ getMUBUFAtomicIns<addrKind, vdataClass, 0, hasRestrictedSOffset, RsrcRC>.ret,
getMUBUFAsmOps<addrKind>.ret,
pattern> {
- let PseudoInstr = opName # "_" # getAddrName<addrKind>.ret;
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # "_" # getAddrName<addrKind>.ret;
let glc_value = 0;
let dlc_value = 0;
let sccb_value = 0;
@@ -836,13 +866,16 @@ class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
RegisterOperand vdata_op,
bit hasRestrictedSOffset = 0,
+ DAGOperand RsrcRC = SReg_128_XNULL,
list<dag> pattern=[]>
: MUBUF_Atomic_Pseudo<opName, addrKind,
(outs vdata_op:$vdata),
- getMUBUFAtomicIns<addrKind, vdata_op, 1, hasRestrictedSOffset>.ret,
+ getMUBUFAtomicIns<addrKind, vdata_op, 1, hasRestrictedSOffset, RsrcRC>.ret,
getMUBUFAsmOps<addrKind>.ret,
pattern> {
- let PseudoInstr = opName # "_rtn_" # getAddrName<addrKind>.ret;
+ let isIndexed = !ne(RsrcRC, SReg_128_XNULL);
+ defvar suffix = !if(isIndexed, "_indexed", "");
+ let PseudoInstr = opName # suffix # "_rtn_" # getAddrName<addrKind>.ret;
let glc_value = 1;
let dlc_value = 0;
let sccb_value = 0;
@@ -862,13 +895,17 @@ multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>;
def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;
- def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>,
- MUBUFAddr64Table <0, NAME # "_VBUFFER">;
- def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>,
- MUBUFAddr64Table <1, NAME # "_VBUFFER">;
- def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;
- def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;
- def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
+ foreach Infix = ["_indexed", ""] in {
+ defvar RsrcRC = !if(!eq(Infix, ""), SReg_128_XNULL, RsrcReg128Op);
+
+ def _VBUFFER # Infix # _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, RsrcRC>,
+ MUBUFAddr64Table <0, NAME # "_VBUFFER" # Infix>;
+ def _VBUFFER # Infix # _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, RsrcRC>,
+ MUBUFAddr64Table <1, NAME # "_VBUFFER" # Infix>;
+ def _VBUFFER # Infix # _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1, RsrcRC>;
+ def _VBUFFER # Infix # _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1, RsrcRC>;
+ def _VBUFFER # Infix # _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1, RsrcRC>;
+ } // end foreach Infix
}
}
@@ -891,6 +928,14 @@ multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>;
def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>;
def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
+
+ def _VBUFFER_indexed_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, RsrcReg128Op>,
+ MUBUFAddr64Table <0, NAME # "_VBUFFER_indexed_RTN">;
+ def _VBUFFER_indexed_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, RsrcReg128Op>,
+ MUBUFAddr64Table <1, NAME # "_VBUFFER_indexed_RTN">;
+ def _VBUFFER_indexed_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1, RsrcReg128Op>;
+ def _VBUFFER_indexed_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1, RsrcReg128Op>;
+ def _VBUFFER_indexed_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1, RsrcReg128Op>;
}
}
@@ -2600,7 +2645,7 @@ class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> :
bits<8> vaddr;
bits<10> vdata;
- bits<7> srsrc;
+ bits<9> srsrc;
bits<7> soffset;
bits<6> cpol;
@@ -2608,8 +2653,7 @@ class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> :
let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);
let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?);
- let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);
- let Inst{49-48} = 0b00;
+ let Inst{49-41} = !if(ps.has_srsrc, srsrc, ?);
let Inst{6-0} = !if(ps.has_soffset, soffset, ?);
let Inst{21-14} = op;
let Inst{22} = ps.tfe;
@@ -2654,25 +2698,29 @@ multiclass VBUFFER_MTBUF_Real_gfx12<bits<4> op, string real_name> {
}
}
-class VBUFFER_Real_gfx13<bits<8> op, BUF_Pseudo ps, string real_name> :
+class VBUFFER_Real_gfx13<bits<8> op, BUF_Pseudo ps, string real_name, bit isIndexed = 0> :
VBUFFER_Real<op, ps, real_name>,
SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX13> {
let AssemblerPredicate = isGFX13Only;
let DecoderNamespace = "GFX13";
+ // For the indexed form the two MSBs of the 9-bit srsrc field select the kind
+ // of index register; for the non-indexed form they are always 0.
+ let Inst{49-48} = !if(isIndexed, !if(ps.has_srsrc, srsrc{8-7}, ?), 0x0);
+ let Inst{47-41} = !if(ps.has_srsrc, srsrc{6-0}, ?);
}
-multiclass VBUFFER_MUBUF_Real_gfx13<bits<8> op, string real_name> {
+multiclass VBUFFER_MUBUF_Real_gfx13<bits<8> op, string real_name, bit isIndexed = 0> {
defvar ps = !cast<MUBUF_Pseudo>(NAME);
- def _gfx13 : VBUFFER_Real_gfx13<op, ps, real_name> {
+ def _gfx13 : VBUFFER_Real_gfx13<op, ps, real_name, isIndexed> {
// Set the format field to be 1 to avoid round-trip issues, as some tools
// print BUF_FMT_INVALID for format 0.
let Inst{61-55} = 0b0000001;
}
}
-multiclass VBUFFER_MTBUF_Real_gfx13<bits<8> op, string real_name> {
+multiclass VBUFFER_MTBUF_Real_gfx13<bits<8> op, string real_name, bit isIndexed = 0> {
defvar ps = !cast<MTBUF_Pseudo>(NAME);
- def _gfx13 : VBUFFER_Real_gfx13<op, ps, real_name> {
+ def _gfx13 : VBUFFER_Real_gfx13<op, ps, real_name, isIndexed> {
bits<7> format;
let Inst{61-55} = format;
}
@@ -2689,6 +2737,8 @@ multiclass MUBUF_Real_AllAddr_gfx13<bits<8> op, string real_name = get_BUF_ps<NA
foreach mode = ["_BOTHEN", "_IDXEN", "_OFFEN", "_OFFSET"] in {
defm _VBUFFER # mode : VBUFFER_MUBUF_Real_gfx13<op, real_name>;
defm _TFE_VBUFFER # mode : VBUFFER_MUBUF_Real_gfx13<op, real_name>;
+ defm _VBUFFER_indexed # mode : VBUFFER_MUBUF_Real_gfx13<op, real_name, 1>;
+ defm _TFE_VBUFFER_indexed # mode : VBUFFER_MUBUF_Real_gfx13<op, real_name, 1>;
}
defvar ps = get_BUF_ps<NAME>;
if !ne(ps.Mnemonic, real_name) then
@@ -2700,6 +2750,7 @@ multiclass MUBUF_Real_Atomics_gfx13<bits<8> op, string real_name = get_BUF_ps<NA
foreach mode = ["_BOTHEN", "_IDXEN", "_OFFEN", "_OFFSET"] in {
foreach rtn = ["", "_RTN"] in {
defm _VBUFFER # mode # rtn : VBUFFER_MUBUF_Real_gfx13<op, real_name>;
+ defm _VBUFFER_indexed # mode # rtn : VBUFFER_MUBUF_Real_gfx13<op, real_name, 1>;
}
}
defvar ps = get_BUF_ps<NAME>;
@@ -3188,6 +3239,7 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
multiclass MTBUF_Real_AllAddr_gfx13_Impl<bits<8> op, string real_name> {
foreach mode = ["_BOTHEN", "_IDXEN", "_OFFEN", "_OFFSET"] in {
defm _VBUFFER # mode : VBUFFER_MTBUF_Real_gfx13<op, real_name>;
+ defm _VBUFFER_indexed # mode : VBUFFER_MTBUF_Real_gfx13<op, real_name, 1>;
}
defvar ps = get_BUF_ps<NAME>;
if !ne(ps.Mnemonic, real_name) then
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 45c1350c146da..ab62086fdcdd3 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -233,6 +233,28 @@ static DecodeStatus decodeSrcOp(MCInst &Inst, unsigned EncSize,
return addOperand(Inst, DAsm->decodeSrcOp(Inst, OpWidth, EncImm));
}
+// Decode an indexed-resource (rsrcidx) 9-bit srsrc field into a 32-bit index
+// register. SGPRs are encoded as 128-251, VGPRs have bit 8 set.
+static DecodeStatus decodeRsrcRegOp(MCInst &Inst, unsigned Imm,
+ uint64_t /* Addr */,
+ const MCDisassembler *Decoder,
+ unsigned OpWidth) {
+ // Uniform-indexed resource. SGPR[0..123] encoded as 128-251.
+ if (Imm >= 128 && Imm < 256)
+ Imm -= 128;
+ return decodeSrcOp(Inst, 9, OpWidth, Imm, Imm, Decoder);
+}
+
+static DecodeStatus decodeRsrcReg128(MCInst &Inst, unsigned Imm,
+ uint64_t /* Addr */,
+ const MCDisassembler *Decoder) {
+ unsigned OpWidth = 32;
+ // 0-127: Uniform-direct resource in SGPRs (SReg_128).
+ if (Imm < 128)
+ OpWidth = 128;
+ return decodeRsrcRegOp(Inst, Imm, 0, Decoder, OpWidth);
+}
+
// Decoder for registers. Imm(7-bit) is number of register, uses decodeSrcOp to
// get register class. Used by SGPR only operands.
#define DECODE_OPERAND_SREG_7(RegClass, OpWidth) \
@@ -241,6 +263,9 @@ static DecodeStatus decodeSrcOp(MCInst &Inst, unsigned EncSize,
#define DECODE_OPERAND_SREG_8(RegClass, OpWidth) \
DECODE_SrcOp(Decode##RegClass##RegisterClass, 8, OpWidth, Imm)
+#define DECODE_OPERAND_SREG_9(RegClass, OpWidth) \
+ DECODE_SrcOp(Decode##RegClass##RegisterClass, 9, OpWidth, Imm)
+
// Decoder for registers. Imm(10-bit): Imm{7-0} is number of register,
// Imm{9} is acc(agpr or vgpr) Imm{8} should be 0 (see VOP3Pe_SMFMAC).
// Set Imm{8} to 1 (IS_VGPR) to decode using 'enum10' from decodeSrcOp.
@@ -334,13 +359,16 @@ DECODE_OPERAND_SREG_7(SReg_64_XEXEC, 64)
DECODE_OPERAND_SREG_7(SReg_64_XEXEC_XNULL, 64)
DECODE_OPERAND_SREG_7(SReg_96, 96)
DECODE_OPERAND_SREG_7(SReg_128, 128)
-DECODE_OPERAND_SREG_7(SReg_128_XNULL, 128)
DECODE_OPERAND_SREG_7(SReg_256, 256)
DECODE_OPERAND_SREG_7(SReg_256_XNULL, 256)
DECODE_OPERAND_SREG_7(SReg_512, 512)
DECODE_OPERAND_SREG_8(SReg_64, 64)
+// GFX13 VBUFFER instructions use a 9-bit srsrc field. For the non-indexed form
+// the two extra MSBs are always 0, so the value still decodes to an SReg_128.
+DECODE_OPERAND_SREG_9(SReg_128_XNULL, 128)
+
DECODE_OPERAND_REG_8(AGPR_32)
DECODE_OPERAND_REG_8(AReg_64)
DECODE_OPERAND_REG_8(AReg_128)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index 2764ff2d68ce0..782c65ef6e276 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -387,6 +387,18 @@ static MCRegister getRegFromMIA(MCRegister Reg, unsigned OpNo,
return Reg;
}
+void AMDGPUInstPrinter::printRsrcReg(const MCInst *MI, unsigned OpNo,
+ const MCSubtargetInfo &STI,
+ raw_ostream &O) {
+ MCRegister Reg = MI->getOperand(OpNo).getReg();
+ bool IsIndexReg = AMDGPU::isRsrcIndexReg(Reg, MRI);
+ if (IsIndexReg)
+ O << "rsrcidx(";
+ printRegOperand(Reg, MI->getOpcode(), OpNo, O, MRI);
+ if (IsIndexReg)
+ O << ')';
+}
+
void AMDGPUInstPrinter::printRegOperand(MCRegister Reg, raw_ostream &O,
const MCRegisterInfo &MRI) {
#if !defined(NDEBUG)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 5e9ebc6716c7f..23da09db265b7 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -71,6 +71,8 @@ class AMDGPUInstPrinter : public MCInstPrinter {
const MCSubtargetInfo &STI, raw_ostream &O);
void printSymbolicFormat(const MCInst *MI,
const MCSubtargetInfo &STI, raw_ostream &O);
+ void printRsrcReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
+ raw_ostream &O);
void printRegOperand(MCRegister Reg, raw_ostream &O);
void printVOPDst(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index 0fd174bcde297..e7dbdad46f8dc 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -57,6 +57,10 @@ class AMDGPUMCCodeEmitter : public MCCodeEmitter {
SmallVectorImpl<MCFixup> &Fixups,
const MCSubtargetInfo &STI) const;
+ void getMachineOpValueRsrcRegOp(const MCInst &MI, unsigned OpNo, APInt &Op,
+ SmallVectorImpl<MCFixup> &Fixups,
+ const MCSubtargetInfo &STI) const;
+
/// Use a fixup to encode the simm16 field for SOPP branch
/// instructions.
void getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
@@ -681,6 +685,22 @@ void AMDGPUMCCodeEmitter::getMachineOpValueT16Lo128(
getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
}
+// Encode an indexed-resource (rsrcidx) operand into the 9-bit srsrc field:
+// bit 8 selects a VGPR (or AGPR) index register, bit 7 selects a 32-bit SGPR
+// index register, and bits 6-0 hold the register index.
+void AMDGPUMCCodeEmitter::getMachineOpValueRsrcRegOp(
+ const MCInst &MI, unsigned OpNo, APInt &Op,
+ SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
+ const MCOperand &MO = MI.getOperand(OpNo);
+ bool IsSReg32 =
+ MRI.getRegClass(AMDGPU::SReg_32RegClassID).contains(MO.getReg());
+ unsigned Enc = MRI.getEncodingValue(MO.getReg());
+ unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
+ bool IsVGPROrAGPR =
+ Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
+ Op = Idx | IsVGPROrAGPR << 8 | IsSReg32 << 7;
+}
+
void AMDGPUMCCodeEmitter::getMachineOpValueCommon(
const MCInst &MI, const MCOperand &MO, unsigned OpNo, APInt &Op,
SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index a410ffedc784d..722178d6cb111 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -1177,6 +1177,32 @@ def VS_128_Align2 : SIRegisterClass<"AMDGPU", VReg_128.RegTypes, 32,
}
} // End GeneratePressureSet = 0
+// Indexed resources (GFX13) encode the buffer/image descriptor as a single
+// 32-bit index register (a Lo256 VGPR for the per-lane/non-uniform form, or an
+// SGPR for the uniform-indexed form) instead of the full SReg_128/SReg_256
+// descriptor.
+def RsrcReg32 : SIRegisterClass<"AMDGPU", Reg32Types.types, 32, (add VGPR_32_Lo256, SGPR_32)> {
+ let isAllocatable = 0;
+ let HasVGPR = 1;
+}
+
+// Parses rsrcidx(<reg>).
+def RsrcRegClass : AsmOperandClass {
+ let Name = "RsrcReg";
+ let ParserMethod = "parseRsrcReg";
+ let PredicateMethod = "isRsrcReg32";
+ let RenderMethod = "addRegOperands";
+}
+
+// Class does not contain any 128 bit values but for disassembler purposes allows
+// decoding to SReg_128.
+def RsrcReg128Op : RegisterOperand<RsrcReg32> {
+ let ParserMatchClass = RsrcRegClass;
+ let DecoderMethod = "decodeRsrcReg128";
+ let EncoderMethod = "getMachineOpValueRsrcRegOp";
+ let PrintMethod = "printRsrcReg";
+}
+
// Define a register tuple class, along with one requiring an even
// aligned base register.
multiclass AVRegClass<int numRegs, list<ValueType> regTypes,
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 9d54b5f0d1c84..46b71cbaa4b9a 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -2701,6 +2701,10 @@ bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
Reg == AMDGPU::SCC;
}
+bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
+ return MRI.getRegClass(AMDGPU::RsrcReg32RegClassID).contains(Reg);
+}
+
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
}
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index e9756fa2a9d7f..1025096966ac2 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1547,6 +1547,10 @@ bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST);
/// Is Reg - scalar register
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI);
+/// \returns true if \p Reg is an indexed-resource index register, i.e. either a
+/// 32-bit SGPR (uniform-indexed form) or a Lo256 VGPR (per-lane indexed form).
+bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI);
+
/// \returns if \p Reg occupies the high 16-bits of a 32-bit register.
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI);
diff --git a/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mtbuf.s b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mtbuf.s
index d60136f7a85ce..11a34e1c90de6 100644
--- a/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mtbuf.s
+++ b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mtbuf.s
@@ -661,3 +661,12 @@ tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_SINT]
tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 scope:SCOPE_SE
// GFX13: tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 scope:SCOPE_SE ; encoding: [0x03,0xc0,0x39,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v5, off, rsrcidx(s8), s3 format:1
+// GFX13: tbuffer_load_format_x v5, off, rsrcidx(s8), s3 ; encoding: [0x03,0x00,0x38,0xc4,0x05,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+tbuffer_load_format_xy v[6:7], off, rsrcidx(v8), s3 format:1
+// GFX13: tbuffer_load_format_xy v[6:7], off, rsrcidx(v8), s3 ; encoding: [0x03,0x40,0x38,0xc4,0x06,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+tbuffer_store_format_x v5, off, rsrcidx(vcc_lo), s3 format:1
+// GFX13: tbuffer_store_format_x v5, off, rsrcidx(vcc_lo), s3 ; encoding: [0x03,0x00,0x39,0xc4,0x05,0xd4,0x81,0x00,0x00,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf.s b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf.s
index e75e943cf9a20..28678a05fc5a3 100644
--- a/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf.s
+++ b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf.s
@@ -2660,3 +2660,67 @@ buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RE
buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
// GFX1250-ERR: :[[@LINE-1]]:23: error: invalid operand for instruction
// GFX13: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, rsrcidx(s8), s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+// GFX13: buffer_load_b32 v5, off, rsrcidx(s8), s3 offset:8388607 ; encoding: [0x03,0x00,0x03,0xc4,0x05,0x10,0x81,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, rsrcidx(v8), s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+// GFX13: buffer_load_b32 v5, off, rsrcidx(v8), s3 offset:8388607 ; encoding: [0x03,0x00,0x03,0xc4,0x05,0x10,0x82,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, rsrcidx(vcc_lo), s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+// GFX13: buffer_load_b32 v5, off, rsrcidx(vcc_lo), s3 offset:8388607 ; encoding: [0x03,0x00,0x03,0xc4,0x05,0xd4,0x81,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, rsrcidx(vcc_hi), s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+// GFX13: buffer_load_b32 v5, off, rsrcidx(vcc_hi), s3 offset:8388607 ; encoding: [0x03,0x00,0x03,0xc4,0x05,0xd6,0x81,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+// GFX13: buffer_load_b64 v[6:7], off, rsrcidx(s8), s3 ; encoding: [0x03,0x40,0x03,0xc4,0x06,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b96 v[6:8], off, rsrcidx(v8), s3
+// GFX1250-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+// GFX13: buffer_load_b96 v[6:8], off, rsrcidx(v8), s3 ; encoding: [0x03,0xc0,0x03,0xc4,0x06,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b128 v[6:9], off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:31: error: invalid operand for instruction
+// GFX13: buffer_load_b128 v[6:9], off, rsrcidx(s8), s3 ; encoding: [0x03,0x80,0x03,0xc4,0x06,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u8 v5, off, rsrcidx(v8), s3
+// GFX1250-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction
+// GFX13: buffer_load_u8 v5, off, rsrcidx(v8), s3 ; encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
+// GFX13: buffer_load_d16_b16 v5, off, rsrcidx(s8), s3 ; encoding: [0x03,0x00,0x09,0xc4,0x05,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, rsrcidx(v8), s3
+// GFX1250-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+// GFX13: buffer_load_d16_hi_u8 v5, off, rsrcidx(v8), s3 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b32 v5, off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:27: error: invalid operand for instruction
+// GFX13: buffer_store_b32 v5, off, rsrcidx(s8), s3 ; encoding: [0x03,0x00,0x07,0xc4,0x05,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b64 v[2:3], off, rsrcidx(v8), s3
+// GFX1250-ERR: :[[@LINE-1]]:31: error: invalid operand for instruction
+// GFX13: buffer_store_b64 v[2:3], off, rsrcidx(v8), s3 ; encoding: [0x03,0x40,0x07,0xc4,0x02,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b8 v5, off, rsrcidx(vcc_lo), s3
+// GFX1250-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction
+// GFX13: buffer_store_b8 v5, off, rsrcidx(vcc_lo), s3 ; encoding: [0x03,0x00,0x06,0xc4,0x05,0xd4,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
+// GFX13: buffer_atomic_add_f32 v5, off, rsrcidx(s8), s3 ; encoding: [0x03,0x00,0x15,0xc4,0x05,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_pk_add_f16 v5, off, rsrcidx(v8), s3
+// GFX1250-ERR: :[[@LINE-1]]:35: error: invalid operand for instruction
+// GFX13: buffer_atomic_pk_add_f16 v5, off, rsrcidx(v8), s3 ; encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x82,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, rsrcidx(s8), s3
+// GFX1250-ERR: :[[@LINE-1]]:40: error: invalid operand for instruction
+// GFX13: buffer_atomic_cmpswap_b32 v[6:7], off, rsrcidx(s8), s3 ; encoding: [0x03,0x40,0x0c,0xc4,0x06,0x10,0x81,0x00,0x00,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf_err.s b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf_err.s
new file mode 100644
index 0000000000000..e2ea7a5965d45
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx13_asm_vbuffer_mubuf_err.s
@@ -0,0 +1,16 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1310 -show-encoding < %s 2>&1 | FileCheck --check-prefix=GFX13-ERR --implicit-check-not=error: %s
+
+buffer_load_b32 v5, off, rsrcidx(s[8:11]), s3 offset:0
+// GFX13-ERR: :[[@LINE-1]]:34: error: rsrcidx operand must be a 32-bit SGPR or VGPR
+
+buffer_load_b32 v5, off, rsrcidx(v[8:11]), s3 offset:0
+// GFX13-ERR: :[[@LINE-1]]:34: error: rsrcidx operand must be a 32-bit SGPR or VGPR
+
+buffer_load_b32 v5, off, rsrcidx s8, s3 offset:0
+// GFX13-ERR: :[[@LINE-1]]:34: error: expected left paren after rsrcidx
+
+buffer_load_b32 v5, off, rsrcidx(s8, s3 offset:0
+// GFX13-ERR: :[[@LINE-1]]:36: error: expected closing parenthesis
+
+buffer_load_b32 v5, off, rsrcidx(off), s3 offset:0
+// GFX13-ERR: :[[@LINE-1]]:34: error: invalid register name
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
index 08701fe348e65..ba42798c5c9be 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_isel.ll.expected
@@ -7,10 +7,10 @@ define i64 @i64_test(i64 %i) nounwind readnone {
; CHECK-NEXT: t0: ch,glue = EntryToken
; CHECK-NEXT: t2: i32,ch = CopyFromReg # D:1 t0, Register:i32 %8
; CHECK-NEXT: t4: i32,ch = CopyFromReg # D:1 t0, Register:i32 %9
-; CHECK-NEXT: t51: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<59>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
+; CHECK-NEXT: t51: i64 = REG_SEQUENCE # D:1 TargetConstant:i32<67>, t2, TargetConstant:i32<3>, t4, TargetConstant:i32<11>
; CHECK-NEXT: t27: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc, align 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
; CHECK-NEXT: t30: i32,ch = BUFFER_LOAD_DWORD_OFFEN<Mem:(dereferenceable load (s32) from %ir.loc + 4, basealign 8, addrspace 5)> TargetFrameIndex:i32<0>, Register:v4i32 $sgpr0_sgpr1_sgpr2_sgpr3, TargetConstant:i32<0>, TargetConstant:i32<4>, TargetConstant:i32<0>, TargetConstant:i1<0>, t0
-; CHECK-NEXT: t33: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<35>, t27, TargetConstant:i32<3>, t30, TargetConstant:i32<11>
+; CHECK-NEXT: t33: v2i32 = REG_SEQUENCE # D:1 TargetConstant:i32<37>, t27, TargetConstant:i32<3>, t30, TargetConstant:i32<11>
; CHECK-NEXT: t10: i64 = V_ADD_U64_PSEUDO # D:1 t51, t33
; CHECK-NEXT: t24: i32 = EXTRACT_SUBREG # D:1 t10, TargetConstant:i32<3>
; CHECK-NEXT: t17: ch,glue = CopyToReg # D:1 t0, Register:i32 $vgpr0, t24
More information about the llvm-commits
mailing list