[llvm] Add RegBankLegalize rules and lowering for G_AMDGPU_S_BUFFER_LOAD (PR #192480)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 09:03:31 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Abhinav Garg (abhigargrepo)
<details>
<summary>Changes</summary>
Add RegBankLegalize rules and lowering for G_AMDGPU_S_BUFFER_LOAD and sub-dword variants (UBYTE, SBYTE, USHORT, SSHORT). The lowering covers all four rsrc/offset divergence combinations:
- Uniform rsrc + uniform offset → scalar SMEM (stays as-is)
- Uniform rsrc + divergent offset → MUBUF (S_BUF_to_BUF, no waterfall)
- Divergent rsrc + uniform offset → SMEM in waterfall loop over rsrc
- Divergent rsrc + divergent offset → MUBUF + waterfall over rsrc
TODO:
1. Fix legalize rule for intrinsic amdgcn_cvt_pkrtz to emit scalar cvt operation in _/AMDGPU/scalar-float-sop2.ll_
2. Fix offset for GFX1250 in _/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll_ as done by PR#<!-- -->178389
---
Patch is 751.66 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/192480.diff
13 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+5)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp (+181-3)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h (+4)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp (+69)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h (+5)
- (modified) llvm/lib/Target/AMDGPU/SMInstructions.td (+1)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll (+27-24)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.buffer.load.ll (+574-771)
- (added) llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.s.buffer.load.ll (+1816)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-s-buffer-load.mir (+21-23)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll (+1581-2393)
- (modified) llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll (+188-84)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 4e956db103188..282a4b740a411 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -7497,6 +7497,11 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
// The 8-bit and 16-bit scalar buffer load instructions have 32-bit
// destination register.
Dst = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32));
+ } else if (Size < 32) {
+ // No native sub-dword scalar buffer load on this subtarget. Load a full
+ // dword into a fresh s32 register — s8/s16 are not valid SGPR types.
+ Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
+ Dst = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32));
} else {
Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
Dst = OrigDst;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 63a940ab5d29b..dc12581a95b89 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -300,6 +300,73 @@ bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
return true;
}
+// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
+// the three offsets (voffset, soffset and instoffset)
+unsigned RegBankLegalizeHelper::setBufferOffsets(
+ MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
+ Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
+ const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
+ const SIInstrInfo &TII = *ST.getInstrInfo();
+ if (std::optional<int64_t> Imm =
+ getIConstantVRegSExtVal(CombinedOffset, MRI)) {
+ uint32_t SOffset, ImmOffset;
+ if (TII.splitMUBUFOffset(*Imm, SOffset, ImmOffset, Alignment)) {
+ VOffsetReg = B.buildConstant({VgprRB, S32}, 0).getReg(0);
+ SOffsetReg = B.buildConstant({SgprRB, S32}, SOffset).getReg(0);
+ InstOffsetVal = ImmOffset;
+ return SOffset + ImmOffset;
+ }
+ }
+ Register Base;
+ unsigned Offset;
+ std::tie(Base, Offset) =
+ AMDGPU::getBaseWithConstantOffset(MRI, CombinedOffset);
+ uint32_t SOffset, ImmOffset;
+ if ((int)Offset > 0 &&
+ TII.splitMUBUFOffset(Offset, SOffset, ImmOffset, Alignment)) {
+ if (Base.isValid() && MRI.getRegBank(Base) == VgprRB) {
+ VOffsetReg = Base;
+ SOffsetReg = B.buildConstant({SgprRB, S32}, SOffset).getReg(0);
+ InstOffsetVal = ImmOffset;
+ return 0;
+ }
+ // If we have SGPR base, we can use it for soffset.
+ if (SOffset == 0) {
+ VOffsetReg = B.buildConstant({VgprRB, S32}, 0).getReg(0);
+ SOffsetReg = Base;
+ InstOffsetVal = ImmOffset;
+ return 0;
+ }
+ }
+ // Handle the variable sgpr + vgpr case.
+ MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, MRI);
+ if (Add && (int)Offset >= 0) {
+ Register Src0 = getSrcRegIgnoringCopies(Add->getOperand(1).getReg(), MRI);
+ Register Src1 = getSrcRegIgnoringCopies(Add->getOperand(2).getReg(), MRI);
+ const RegisterBank *Src0Bank = MRI.getRegBank(Src0);
+ const RegisterBank *Src1Bank = MRI.getRegBank(Src1);
+ if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
+ VOffsetReg = Src0;
+ SOffsetReg = Src1;
+ return 0;
+ }
+ if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
+ VOffsetReg = Src1;
+ SOffsetReg = Src0;
+ return 0;
+ }
+ }
+ // Ensure we have a VGPR for the combined offset. This could be an issue if we
+ // have an SGPR offset and a VGPR resource.
+ if (MRI.getRegBank(CombinedOffset) == VgprRB) {
+ VOffsetReg = CombinedOffset;
+ } else {
+ VOffsetReg = B.buildCopy({VgprRB, S32}, CombinedOffset).getReg(0);
+ }
+ SOffsetReg = B.buildConstant({SgprRB, S32}, 0).getReg(0);
+ return 0;
+}
+
bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
MachineFunction &MF = B.getMF();
@@ -1205,6 +1272,109 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerSplitTo32Select(MI);
case SplitTo32SExtInReg:
return lowerSplitTo32SExtInReg(MI);
+ case S_BUF_to_BUF: {
+ Register Dst = MI.getOperand(0).getReg();
+ LLT Ty = MRI.getType(Dst);
+ const RegisterBank *RSrcBank = MRI.getRegBank(MI.getOperand(1).getReg());
+ // FIXME: 96-bit case was widened during legalize. We need to narrow it
+ // back here but don't have an MMO.
+ unsigned LoadSize = Ty.getSizeInBits();
+ int NumLoads = 1;
+ if (LoadSize == 256 || LoadSize == 512) {
+ NumLoads = LoadSize / 128;
+ Ty = Ty.divide(NumLoads);
+ }
+ // Use the alignment to ensure that the required offsets will fit into the
+ // immediate offsets.
+ const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
+ MachineFunction &MF = B.getMF();
+ Register SOffset;
+ Register VOffset;
+ int64_t ImmOffset = 0;
+ unsigned MMOOffset = setBufferOffsets(B, MI.getOperand(2).getReg(), VOffset,
+ SOffset, ImmOffset, Alignment);
+ // TODO: 96-bit loads were widened to 128-bit results. Shrink the
+ // result if we can, but we need to track an MMO for that.
+ // const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
+ const unsigned MemSize = divideCeil(Ty.getSizeInBits(), 8);
+ const DataLayout DL = MF.getDataLayout();
+ Type *IRTy = getTypeForLLT(Ty, MF.getFunction().getContext());
+ const Align MemAlign(DL.getABITypeAlign(IRTy));
+ MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
+ MachinePointerInfo(),
+ MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOInvariant,
+ MemSize, MemAlign);
+ if (MMOOffset != 0)
+ BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset, MemSize);
+ // If only the offset is divergent, emit a MUBUF buffer load
+ // instead. We can assume that the buffer is unswizzled.
+ Register RSrc = MI.getOperand(1).getReg();
+ Register VIndex = B.buildConstant(S32, 0).getReg(0);
+ B.getMRI()->setRegBank(VIndex, *VgprRB);
+ SmallVector<Register, 4> LoadParts(NumLoads);
+ MachineBasicBlock::iterator MII = MI.getIterator();
+ MachineInstrSpan Span(MII, &B.getMBB());
+ unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
+ switch (MI.getOpcode()) {
+ case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
+ Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
+ break;
+ case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
+ Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
+ break;
+ case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
+ Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
+ break;
+ case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
+ Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
+ break;
+ default:
+ break;
+ }
+ for (int i = 0; i < NumLoads; ++i) {
+ if (NumLoads == 1) {
+ LoadParts[i] = Dst;
+ } else {
+ LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
+ MRI.setRegBank(LoadParts[i], *VgprRB);
+ }
+ if (i != 0)
+ BaseMMO = MF.getMachineMemOperand(BaseMMO, 16, MemSize);
+ B.buildInstr(Opc)
+ .addDef(LoadParts[i]) // vdata
+ .addUse(RSrc) // rsrc
+ .addUse(VIndex) // vindex
+ .addUse(VOffset) // voffset
+ .addUse(SOffset) // soffset
+ .addImm(ImmOffset + 16 * i) // offset(imm)
+ .addImm(0) // cachepolicy, swizzled buffer(imm)
+ .addImm(0) // idxen(imm)
+ .addMemOperand(BaseMMO);
+ }
+ // TODO: If only the resource is a VGPR, it may be better to execute the
+ // scalar load in the waterfall loop if the resource is expected to
+ // frequently be dynamically uniform.
+ if (RSrcBank != SgprRB) {
+ // Remove the original instruction to avoid potentially confusing the
+ // waterfall loop logic.
+ B.setInstr(*Span.begin());
+ MI.eraseFromParent();
+ SmallSet<Register, 4> OpsToWaterfall;
+ OpsToWaterfall.insert(RSrc);
+ executeInWaterfallLoop(B, {OpsToWaterfall, Span.begin(), Span.end()});
+ }
+ if (NumLoads != 1) {
+ if (Ty.isVector())
+ B.buildConcatVectors(Dst, LoadParts);
+ else
+ B.buildMergeLikeInstr(Dst, LoadParts);
+ }
+ // We removed the instruction earlier with a waterfall loop.
+ if (RSrcBank == SgprRB)
+ MI.eraseFromParent();
+ break;
+ }
case SplitLoad: {
LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
unsigned Size = DstTy.getSizeInBits();
@@ -1402,6 +1572,8 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
case UniInVgprS64:
case Sgpr64ToVgprDst:
return LLT::scalar(64);
+ case Vgpr96:
+ return LLT::scalar(96);
case Sgpr128:
case Vgpr128:
return LLT::scalar(128);
@@ -1495,7 +1667,7 @@ LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
case UniInVgprB128:
if (Ty == LLT::scalar(128) || Ty == LLT::fixed_vector(4, 32) ||
Ty == LLT::fixed_vector(2, 64) || Ty == LLT::fixed_vector(8, 16) ||
- isAnyPtr(Ty, 128))
+ Ty == LLT::fixed_vector(2, LLT::pointer(1, 64)) || isAnyPtr(Ty, 128))
return Ty;
return LLT();
case VgprB160:
@@ -1507,14 +1679,17 @@ LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
case VgprB256:
case UniInVgprB256:
if (Ty == LLT::scalar(256) || Ty == LLT::fixed_vector(8, 32) ||
- Ty == LLT::fixed_vector(4, 64) || Ty == LLT::fixed_vector(16, 16))
+ Ty == LLT::fixed_vector(4, 64) || Ty == LLT::fixed_vector(16, 16) ||
+ Ty == LLT::fixed_vector(4, LLT::pointer(1, 64)) ||
+ Ty == LLT::fixed_vector(8, LLT::pointer(3, 32)))
return Ty;
return LLT();
case SgprB512:
case VgprB512:
case UniInVgprB512:
if (Ty == LLT::scalar(512) || Ty == LLT::fixed_vector(16, 32) ||
- Ty == LLT::fixed_vector(8, 64))
+ Ty == LLT::fixed_vector(32, 16) || Ty == LLT::fixed_vector(8, 64) ||
+ Ty == LLT::fixed_vector(8, LLT::pointer(1, 64)))
return Ty;
return LLT();
case SgprBRC: {
@@ -1595,6 +1770,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
case Vgpr16:
case Vgpr32:
case Vgpr64:
+ case Vgpr96:
case Vgpr128:
case VgprP0:
case VgprP1:
@@ -1662,6 +1838,7 @@ bool RegBankLegalizeHelper::applyMappingDst(
case Vgpr16:
case Vgpr32:
case Vgpr64:
+ case Vgpr96:
case Vgpr128:
case VgprP0:
case VgprP1:
@@ -1855,6 +2032,7 @@ bool RegBankLegalizeHelper::applyMappingSrc(
case Vgpr16:
case Vgpr32:
case Vgpr64:
+ case Vgpr96:
case Vgpr128:
case VgprP0:
case VgprP1:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 57df192832dc6..a127a64eeb3fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -115,6 +115,10 @@ class RegBankLegalizeHelper {
const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
WaterfallInfo &WFI);
+ unsigned setBufferOffsets(MachineIRBuilder &B, Register CombinedOffset,
+ Register &VOffsetReg, Register &SOffsetReg,
+ int64_t &InstOffsetVal, Align Alignment);
+
bool splitLoad(MachineInstr &MI, ArrayRef<LLT> LLTBreakdown,
LLT MergeTy = LLT());
bool widenLoad(MachineInstr &MI, LLT WideTy, LLT MergeTy = LLT());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 8028a4cb7c878..e07c61228ed4b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -132,6 +132,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isUniform(Reg);
case UniV2S32:
return MRI.getType(Reg) == LLT::fixed_vector(2, 32) && MUI.isUniform(Reg);
+ case UniV4S32:
+ return MRI.getType(Reg) == LLT::fixed_vector(4, 32) && MUI.isUniform(Reg);
case UniB32:
return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniform(Reg);
case UniB64:
@@ -165,6 +167,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg) == LLT::scalar(32) && MUI.isDivergent(Reg);
case DivS64:
return MRI.getType(Reg) == LLT::scalar(64) && MUI.isDivergent(Reg);
+ case DivS96:
+ return MRI.getType(Reg) == LLT::scalar(96) && MUI.isDivergent(Reg);
case DivS128:
return MRI.getType(Reg) == LLT::scalar(128) && MUI.isDivergent(Reg);
case DivP0:
@@ -189,6 +193,10 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isDivergent(Reg);
case DivV2S32:
return MRI.getType(Reg) == LLT::fixed_vector(2, 32) && MUI.isDivergent(Reg);
+ case DivV4S32:
+ return MRI.getType(Reg) == LLT::fixed_vector(4, 32) && MUI.isDivergent(Reg);
+ case DivV2S64:
+ return MRI.getType(Reg) == LLT::fixed_vector(2, 64) && MUI.isDivergent(Reg);
case DivV3S32:
return MRI.getType(Reg) == LLT::fixed_vector(3, 32) && MUI.isDivergent(Reg);
case DivV4S16:
@@ -1179,6 +1187,67 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+ addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
+ // waterfall expansion is part of S_BUF_to_BUF
+ .Any({{UniB32}, {{SgprB32}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB32, UniV4S32, DivB32},
+ {{VgprB32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB32, DivV4S32, UniB32},
+ {{VgprB32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB32, DivV4S32, DivB32},
+ {{VgprB32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+
+ .Any({{UniB64}, {{SgprB64}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB64, UniV4S32, DivB32},
+ {{VgprB64}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB64, DivV4S32, UniB32},
+ {{VgprB64}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB64, DivV4S32, DivB32},
+ {{VgprB64}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+
+ .Any({{UniB96}, {{SgprB96}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB96, UniV4S32, DivB32},
+ {{VgprB96}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB96, DivV4S32, UniB32},
+ {{VgprB96}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB96, DivV4S32, DivB32},
+ {{VgprB96}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+
+ .Any({{UniB128}, {{SgprB128}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB128, UniV4S32, DivB32},
+ {{VgprB128}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB128, DivV4S32, UniB32},
+ {{VgprB128}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB128, DivV4S32, DivB32},
+ {{VgprB128}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+
+ .Any({{UniB256}, {{SgprB256}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB256, UniV4S32, DivB32},
+ {{VgprB256}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB256, DivV4S32, UniB32},
+ {{VgprB256}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB256, DivV4S32, DivB32},
+ {{VgprB256}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+
+ .Any({{UniB512}, {{SgprB512}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivB512, UniV4S32, DivB32},
+ {{VgprB512}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB512, DivV4S32, UniB32},
+ {{VgprB512}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivB512, DivV4S32, DivB32},
+ {{VgprB512}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
+
+ addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD_SBYTE, G_AMDGPU_S_BUFFER_LOAD_UBYTE,
+ G_AMDGPU_S_BUFFER_LOAD_SSHORT,
+ G_AMDGPU_S_BUFFER_LOAD_USHORT})
+ .Any({{UniS32, UniV4S32, UniS32}, {{Sgpr32}, {SgprV4S32, Sgpr32}}})
+ .Any({{DivS32, UniV4S32, DivS32},
+ {{Vgpr32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
+ .Any({{DivS32, DivV4S32, UniS32},
+ {{Vgpr32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
+ .Any({{DivS32, DivV4S32, DivS32},
+ {{Vgpr32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
+
addRulesForGOpcs({G_AMDGPU_BUFFER_STORE, G_AMDGPU_BUFFER_STORE_BYTE,
G_AMDGPU_BUFFER_STORE_SHORT, G_AMDGPU_BUFFER_STORE_FORMAT,
G_AMDGPU_BUFFER_STORE_FORMAT_D16,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 3d282220d8611..1e422c87b48ba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -56,6 +56,7 @@ enum UniformityLLTOpPredicateID {
DivS16,
DivS32,
DivS64,
+ DivS96,
DivS128,
// pointers
@@ -100,10 +101,12 @@ enum UniformityLLTOpPredicateID {
UniV2S16,
UniV2S32,
+ UniV4S32,
UniV2S64,
DivV2S16,
DivV2S32,
+ DivV4S32,
DivV2S64,
DivV3S32,
DivV4S16,
@@ -178,6 +181,7 @@ enum RegBankLLTMappingApplyID {
Vgpr16,
Vgpr32,
Vgpr64,
+ Vgpr96,
Vgpr128,
VgprP0,
VgprP1,
@@ -279,6 +283,7 @@ enum LoweringMethodID {
ScalarizeToS16,
SplitTo32Select,
SplitTo32SExtInReg,
+ S_BUF_to_BUF,
Ext32To64,
UniCstExt,
SplitLoad,
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index ee8d29c77708b..09e7a33bc84a4 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -1080,6 +1080,7 @@ defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>;
defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>;
defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>;
+defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", i64>;
defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX3", v3i32, false>;
defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>;
defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll
index e3b92508c2997..6cd78470e169e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -global-isel < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -global-isel -new-reg-bank-select < %s | FileCheck %s
; early-tailduplication deletes cycle exit block created by structurize-cfg
; that had exactly one predecessor. Now, new cycle exit block has two
@@ -8,25 +8,25 @@
define amdgpu_ps void @_amdgpu_ps_main(i1 %arg) {
; CHECK-LABEL: _amdgpu_ps_main:
; CHECK: ; %bb.0: ; %bb
-; CHECK-NEXT: s_mov_b32 s4, 0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: s_mov_b32 s5, s4
-; CHECK-NEXT: s_mov_b32 s6, s4
-; CHECK-NEXT: s_mov_b32 s7, s4
; CHECK-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
-; CHECK-NEXT: s_buffer_load_dword s1, s[4:7], 0x0
; CHECK-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
; CHECK-NEXT: s_mov_b32 s10, -1
; CHECK-NEXT: s_mov_b32 s11, 0x31c16000
; CHECK-NEXT: s_add_u32 s8, s8, s0
-; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, v0
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: s_mov_b32 s2, s0
+; CHECK-NEXT: s_mov_b32 s3, s0
; CHECK-NEXT: s_addc_u32 s9, s9, 0
+; CHECK-NEXT: s_buffer_load_dword s1, s[0:3], 0x0
; CHECK-NEXT: s_mov_b32 s32, 0
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_cmp_ge_i32 s1, 0
; CHECK-NEXT: s_cbranch_scc0 .LBB0_2
; CHECK-NEXT: .LBB0_1: ; %bb12
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s4
+; CHECK-N...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/192480
More information about the llvm-commits
mailing list