[llvm] [AMDGPU] Split large raw buffer offsets to SOFFSET where allowed (PR #207821)
Krzysztof Drewniak via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 11:10:38 PDT 2026
https://github.com/krzysz00 updated https://github.com/llvm/llvm-project/pull/207821
>From 9e4b95f573b562bc83beb68a4284801f85cfc878 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Wed, 24 Jun 2026 21:51:42 +0000
Subject: [PATCH 1/3] [AMDGPU] Split large raw buffer offsets to SOFFSET where
allowed
If we're not doing vindex or swizzling and aren't very old (that is,
we're not in a case where the buffer OOB formula is
`voffset +[checked] imm + payload > num_records -[saturating]
soffset`) or a politer version on gfx1250 *and* the large constant
offset is added to voffset (if any) in a NUW like way *and* the add to
SOFFSET wouldn't overflow (so, in practice, if SOFFSET is a constant
for now), then we can safely move the part that doesn't fit into the
instruction immediate into the SOFFSET field instead of the VOFFSET
field, saving on VGPRs.
This commit implements this change - and, while we're here, makes
GlobalIsel match SelectionDAG in terms of recognizing an `or disjoint`
as add-like.
Note that the exclusion of SEA_ISLANDS and older is taken from
selectMUBEFOffsets (the s_buffer_load version) just to be safe.
AI disclosure: AI wrote this and I poked it into shape a bunch.
Co-Authored-By: Codex <codex at openai.com>
---
.../Target/AMDGPU/AMDGPUGlobalISelUtils.cpp | 8 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 116 ++++--
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 12 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 60 ---
.../Target/AMDGPU/AMDGPURegisterBankInfo.h | 3 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 166 +++++---
llvm/lib/Target/AMDGPU/SIISelLowering.h | 17 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 13 +-
...llvm.amdgcn.raw.ptr.buffer.offset-split.ll | 13 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 19 +-
.../AMDGPU/llvm.amdgcn.raw.buffer.load.ll | 198 ++++++----
.../AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll | 353 +++++++++++++++---
.../llvm.amdgcn.raw.ptr.tbuffer.load.ll | 112 +++---
.../llvm.amdgcn.raw.ptr.tbuffer.store.ll | 100 +++--
.../AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll | 116 +++---
.../AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll | 104 ++++--
16 files changed, 928 insertions(+), 482 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index f36935d8c0e8f..2bfeb0546e1af 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -46,7 +46,6 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
return std::pair(Reg, 0);
}
- // TODO: Handle G_OR used for add case
if (mi_match(Def->getOperand(2).getReg(), MRI, m_ICst(Offset)))
return std::pair(Def->getOperand(1).getReg(), Offset);
@@ -56,9 +55,10 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
}
Register Base;
- if (ValueTracking && mi_match(Reg, MRI, m_GOr(m_Reg(Base), m_ICst(Offset))) &&
- ValueTracking->maskedValueIsZero(Base,
- APInt(32, Offset, /*isSigned=*/true)))
+ if (mi_match(Reg, MRI, m_GOr(m_Reg(Base), m_ICst(Offset))) &&
+ (Def->getFlag(MachineInstr::Disjoint) ||
+ (ValueTracking && ValueTracking->maskedValueIsZero(
+ Base, APInt(32, Offset, /*isSigned=*/true)))))
return std::pair(Base, Offset);
// Handle G_PTRTOINT (G_PTR_ADD base, const) case
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c57ef9392a4ca..7281bec93850b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -36,6 +36,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Support/CheckedArithmetic.h"
#define DEBUG_TYPE "amdgpu-legalinfo"
@@ -6493,60 +6494,115 @@ bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
return true;
}
+static bool canMoveBufferOffsetToSOffset(const GCNSubtarget &ST,
+ bool HasLinearOOB, bool IsNUW,
+ uint32_t Offset) {
+ // There is a hardware bug in SI and CI which prevents address clamping in
+ // MUBUF instructions from working correctly with SOffsets. The immediate
+ // offset is unaffected.
+ return HasLinearOOB && IsNUW && static_cast<int32_t>(Offset) > 0 &&
+ ST.getGeneration() > AMDGPUSubtarget::SEA_ISLANDS;
+}
+
+static std::optional<uint32_t>
+getCombinedBufferSOffset(MachineRegisterInfo &MRI, Register SOffset,
+ uint32_t Offset) {
+ APInt C;
+ if (!mi_match(SOffset, MRI, m_ICst(C)))
+ return std::nullopt;
+ return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C.getZExtValue()),
+ Offset);
+}
+
// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
-// the instruction's soffset field). This function takes the first kind of
-// offset and figures out how to split it between voffset and immoffset.
-std::pair<Register, unsigned>
+// the instruction's soffset field, except that it does affect num_records and
+// so can (if there's no unsigned overflow) be used for bounds checking in raw.*
+// intrinsics). This function takes both offsets and figures out how to split
+// them between voffset, soffset, and immoffset.
+std::tuple<Register, Register, unsigned>
AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const {
+ Register OrigOffset,
+ Register OrigSOffset, bool HasLinearOOB,
+ GISelValueTracking *VT) const {
const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
Register BaseReg;
unsigned ImmOffset;
const LLT S32 = LLT::scalar(32);
MachineRegisterInfo &MRI = *B.getMRI();
-
+ Register SOffset = OrigSOffset;
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no unsigned
// overflow.
bool CheckNUW = ST.hasGFX1250Insts();
- std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
- MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
+ std::tie(BaseReg, ImmOffset) =
+ AMDGPU::getBaseWithConstantOffset(MRI, OrigOffset, VT, CheckNUW);
+ bool IsNUW = false;
+ if (ImmOffset) {
+ if (!BaseReg) {
+ IsNUW = true;
+ } else if (MachineInstr *OffsetDef =
+ getDefIgnoringCopies(OrigOffset, MRI)) {
+ // Match SelectionDAG: a G_OR with a constant reaches here only when it is
+ // add-like, and such an OR satisfies the no-wrap condition.
+ IsNUW = OffsetDef->getOpcode() == TargetOpcode::G_OR ||
+ (OffsetDef->getOpcode() == TargetOpcode::G_ADD &&
+ OffsetDef->getFlag(MachineInstr::NoUWrap));
+ }
+ }
// If BaseReg is a pointer, convert it to int.
- if (MRI.getType(BaseReg).isPointer())
+ if (BaseReg && MRI.getType(BaseReg).isPointer())
BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
+
+ // If the immediate value is too big for the immoffset field, put only bits
+ // that would normally fit in the immoffset field. The remaining value
+ // copied/added for the voffset field is a large power of 2, and it stands
+ // more chance of being CSEd with the copy/add for another similar
+ // load/store. For eligible raw.(t)buffer.* operations, the branch below may
+ // instead move this overflow to soffset.
+ //
+ // Do not do the rounding-down split for negative numbers, as the addition of
+ // immoffset and voffset is checked for unsigned overflow (which causes the
+ // load/store to be marked OOB) and the soffset is (on gfx8 through gfx12; see
+ // the ISA MUBUF/MTBUF addressing description) subtracted from num_records and
+ // not added to the offset.
if ((int32_t)Overflow < 0) {
Overflow += ImmOffset;
ImmOffset = 0;
}
- if (Overflow != 0) {
- if (!BaseReg) {
- BaseReg = B.buildConstant(S32, Overflow).getReg(0);
- } else {
- auto OverflowVal = B.buildConstant(S32, Overflow);
- BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
+ std::optional<uint32_t> NewSOffset;
+ if (Overflow &&
+ canMoveBufferOffsetToSOffset(ST, HasLinearOOB, IsNUW, Overflow))
+ NewSOffset = getCombinedBufferSOffset(MRI, SOffset, Overflow);
+
+ if (NewSOffset) {
+ // For raw.(t)buffer.* operations with a bare constant or no-wrap add, the
+ // overflow can be added to an existing soffset if the addition would not
+ // cause unsigned overflow. We conservatively restrict ourselves to
+ // constant soffsets here.
+ SOffset = B.buildConstant(S32, *NewSOffset).getReg(0);
+ } else {
+ if (Overflow != 0) {
+ if (!BaseReg) {
+ BaseReg = B.buildConstant(S32, Overflow).getReg(0);
+ } else {
+ auto OverflowVal = B.buildConstant(S32, Overflow);
+ BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
+ }
}
}
if (!BaseReg)
BaseReg = B.buildConstant(S32, 0).getReg(0);
- return std::pair(BaseReg, ImmOffset);
+ return {BaseReg, SOffset, ImmOffset};
}
/// Handle register layout difference for f16 images for some subtargets.
@@ -6694,7 +6750,8 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
unsigned Opc;
if (IsTyped) {
@@ -6832,7 +6889,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const bool Unpacked = ST.hasUnpackedD16VMem();
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
unsigned Opc;
@@ -7016,8 +7074,9 @@ static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
}
bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
- MachineIRBuilder &B,
+ LegalizerHelper &Helper,
Intrinsic::ID IID) const {
+ MachineIRBuilder &B = Helper.MIRBuilder;
const bool IsCmpSwap =
IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
@@ -7058,7 +7117,8 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
MachineMemOperand *MMO = *MI.memoperands_begin();
unsigned ImmOffset;
- std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ std::tie(VOffset, SOffset, ImmOffset) = splitBufferOffsets(
+ B, VOffset, SOffset, !HasVIndex, Helper.getValueTracking());
auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
.addDef(Dst)
@@ -8511,7 +8571,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
- return legalizeBufferAtomic(MI, B, IntrID);
+ return legalizeBufferAtomic(MI, Helper, IntrID);
case Intrinsic::amdgcn_rsq_clamp:
return legalizeRsqClampIntrinsic(MI, MRI, B);
case Intrinsic::amdgcn_image_bvh_intersect_ray:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 89819fe990f5a..27facfe7b82ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -14,14 +14,16 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H
-#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "AMDGPUArgumentUsageInfo.h"
#include "SIInstrInfo.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
+#include <tuple>
namespace llvm {
class GCNTargetMachine;
class GCNSubtarget;
+class GISelValueTracking;
class MachineIRBuilder;
namespace AMDGPU {
@@ -207,8 +209,10 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, unsigned AddrSpace) const;
- std::pair<Register, unsigned> splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const;
+ std::tuple<Register, Register, unsigned>
+ splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset,
+ Register OrigSOffset, bool HasLinearOOB,
+ GISelValueTracking *VT) const;
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
Register Reg, bool ImageStore = false) const;
@@ -219,7 +223,7 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool IsTyped, bool IsFormat) const;
bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper,
bool IsFormat, bool IsTyped) const;
- bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B,
+ bool legalizeBufferAtomic(MachineInstr &MI, LegalizerHelper &Helper,
Intrinsic::ID IID) const;
bool legalizeBVHIntersectRayIntrinsic(MachineInstr &MI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 200234c23c886..4a3e402f53c83 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1807,66 +1807,6 @@ Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
.getReg(0);
}
-static std::pair<Register, unsigned>
-getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
- int64_t Const;
- if (mi_match(Reg, MRI, m_ICst(Const)))
- return std::pair(Register(), Const);
-
- Register Base;
- if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const))))
- return std::pair(Base, Const);
-
- // TODO: Handle G_OR used for add case
- return std::pair(Reg, 0);
-}
-
-std::pair<Register, unsigned>
-AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
- Register OrigOffset) const {
- const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(Subtarget);
- Register BaseReg;
- unsigned ImmOffset;
- const LLT S32 = LLT::scalar(32);
-
- // TODO: Use AMDGPU::getBaseWithConstantOffset() instead.
- std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(),
- OrigOffset);
-
- unsigned C1 = 0;
- if (ImmOffset != 0) {
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
- unsigned Overflow = ImmOffset & ~MaxImm;
- ImmOffset -= Overflow;
- if (static_cast<int32_t>(Overflow) < 0) {
- Overflow += ImmOffset;
- ImmOffset = 0;
- }
-
- C1 = ImmOffset;
- if (Overflow != 0) {
- if (!BaseReg)
- BaseReg = B.buildConstant(S32, Overflow).getReg(0);
- else {
- auto OverflowVal = B.buildConstant(S32, Overflow);
- BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
- }
- }
- }
-
- if (!BaseReg)
- BaseReg = B.buildConstant(S32, 0).getReg(0);
-
- return {BaseReg, C1};
-}
-
bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
Register SrcReg) const {
MachineRegisterInfo &MRI = *B.getMRI();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index 5f550b426ec09..6ac00008d4d4d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -90,9 +90,6 @@ class AMDGPURegisterBankInfo final : public AMDGPUGenRegisterBankInfo {
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
Register Reg) const;
- std::pair<Register, unsigned>
- splitBufferOffsets(MachineIRBuilder &B, Register Offset) const;
-
/// See RegisterBankInfo::applyMapping.
void applyMappingImpl(MachineIRBuilder &Builder,
const OperandsMapper &OpdMapper) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a29070953ee38..2a01718b1f653 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -44,6 +44,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/CheckedArithmetic.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
@@ -11384,8 +11385,10 @@ SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
SDValue VData = Op.getOperand(2);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
VData, // vdata
@@ -11412,8 +11415,10 @@ SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
SDValue VData = Op.getOperand(2);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
VData, // vdata
@@ -11513,8 +11518,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(3), DAG);
- auto SOffset = selectSOffset(Op.getOperand(4), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(3), Op.getOperand(4), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Rsrc, // rsrc
@@ -11540,8 +11547,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Rsrc, // rsrc
@@ -11560,8 +11569,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
MemSDNode *M = cast<MemSDNode>(Op);
EVT LoadVT = Op.getValueType();
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(3), DAG);
- auto SOffset = selectSOffset(Op.getOperand(4), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(3), Op.getOperand(4), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
@@ -11587,8 +11598,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
MemSDNode *M = cast<MemSDNode>(Op);
EVT LoadVT = Op.getValueType();
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(2), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
@@ -11725,8 +11738,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(4), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Op.getOperand(2), // src
@@ -11749,8 +11764,10 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
SDValue Rsrc = bufferRsrcPtrToVector(Op->getOperand(4), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(6), DAG);
- auto SOffset = selectSOffset(Op.getOperand(7), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(6), Op.getOperand(7), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Op.getOperand(0), // Chain
Op.getOperand(2), // src
@@ -12258,8 +12275,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData, // vdata
@@ -12286,8 +12305,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData, // vdata
@@ -12331,8 +12352,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
}
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
- auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(4), Op.getOperand(5), DAG,
+ /*HasLinearOOB=*/true);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData,
@@ -12382,8 +12405,10 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
}
auto Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
- auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
- auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
+ auto [VOffset, SOffset, Offset] =
+ splitBufferOffsets(Op.getOperand(5), Op.getOperand(6), DAG,
+ /*HasLinearOOB=*/false);
+ SOffset = selectSOffset(SOffset, DAG, Subtarget);
SDValue Ops[] = {
Chain,
VData,
@@ -12771,6 +12796,25 @@ static bool isNoUnsignedWrap(SDValue Addr) {
Addr->getOpcode() == ISD::OR;
}
+static bool canMoveBufferOffsetToSOffset(const GCNSubtarget *Subtarget,
+ bool HasLinearOOB, bool IsNUW,
+ uint32_t Offset) {
+ // There is a hardware bug in SI and CI which prevents address clamping in
+ // MUBUF instructions from working correctly with SOffsets. The immediate
+ // offset is unaffected.
+ return HasLinearOOB && IsNUW && static_cast<int32_t>(Offset) > 0 &&
+ Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS;
+}
+
+static std::optional<uint32_t> getCombinedBufferSOffset(SDValue SOffset,
+ uint32_t Offset) {
+ auto *C = dyn_cast<ConstantSDNode>(SOffset);
+ if (!C)
+ return std::nullopt;
+ return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C->getZExtValue()),
+ Offset);
+}
+
bool SITargetLowering::shouldPreservePtrArith(const Function &F,
EVT PtrVT) const {
return PtrVT == MVT::i64;
@@ -12785,23 +12829,30 @@ bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
-// the instruction's soffset field). This function takes the first kind of
-// offset and figures out how to split it between voffset and immoffset.
-std::pair<SDValue, SDValue>
-SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
+// the instruction's soffset field, except that it does affect num_records and
+// so can (if there's no unsigned overflow) be used for bounds checking in raw.*
+// intrinsics). This function takes both offsets and figures out how to split
+// them between voffset, soffset, and immoffset.
+std::tuple<SDValue, SDValue, SDValue>
+SITargetLowering::splitBufferOffsets(SDValue Offset, SDValue SOffset,
+ SelectionDAG &DAG,
+ bool HasLinearOOB) const {
SDLoc DL(Offset);
const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
SDValue N0 = Offset;
ConstantSDNode *C1 = nullptr;
+ bool IsNUW = false;
- if ((C1 = dyn_cast<ConstantSDNode>(N0)))
+ if ((C1 = dyn_cast<ConstantSDNode>(N0))) {
N0 = SDValue();
- else if (DAG.isBaseWithConstantOffset(N0)) {
+ IsNUW = true;
+ } else if (DAG.isBaseWithConstantOffset(N0)) {
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no
// unsigned overflow.
bool CheckNUW = Subtarget->hasGFX1250Insts();
- if (!CheckNUW || isNoUnsignedWrap(N0)) {
+ IsNUW = isNoUnsignedWrap(N0);
+ if (!CheckNUW || IsNUW) {
C1 = cast<ConstantSDNode>(N0.getOperand(1));
N0 = N0.getOperand(0);
}
@@ -12809,36 +12860,55 @@ SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
if (C1) {
unsigned ImmOffset = C1->getZExtValue();
- // If the immediate value is too big for the immoffset field, put only bits
- // that would normally fit in the immoffset field. The remaining value that
- // is copied/added for the voffset field is a large power of 2, and it
- // stands more chance of being CSEd with the copy/add for another similar
- // load/store.
- // However, do not do that rounding down if that is a negative
- // number, as it appears to be illegal to have a negative offset in the
- // vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
+
+ // If the immediate value is too big for the immoffset field, put only
+ // bits that would normally fit in the immoffset field. The remaining
+ // value copied/added for the voffset field is a large power of 2, and it
+ // stands more chance of being CSEd with the copy/add for another similar
+ // load/store. For eligible raw.(t)buffer.* operations, the branch below
+ // may instead move this overflow to soffset.
+ //
+ // Do not do the rounding-down split for negative numbers, as the addition
+ // of immoffset and voffset is checked for unsigned overflow (which causes
+ // the load/store to be marked OOB) and the soffset is (on gfx8 through
+ // gfx12; see the ISA MUBUF/MTBUF addressing description) subtracted from
+ // num_records and not added to the offset.
if ((int32_t)Overflow < 0) {
Overflow += ImmOffset;
ImmOffset = 0;
}
- C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
- if (Overflow) {
- auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
- if (!N0)
- N0 = OverflowVal;
- else {
- SDValue Ops[] = {N0, OverflowVal};
- N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops);
+
+ std::optional<uint32_t> NewSOffset;
+ if (Overflow &&
+ canMoveBufferOffsetToSOffset(Subtarget, HasLinearOOB, IsNUW, Overflow))
+ NewSOffset = getCombinedBufferSOffset(SOffset, Overflow);
+
+ if (NewSOffset) {
+ // For raw.(t)buffer.* operations with a bare constant or no-wrap add,
+ // the overflow can be added to an existing soffset if the addition would
+ // not cause unsigned overflow. We conservatively restrict ourselves to
+ // constant soffsets here.
+ SOffset = DAG.getConstant(*NewSOffset, DL, MVT::i32);
+ } else {
+ if (Overflow) {
+ auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
+ if (!N0)
+ N0 = OverflowVal;
+ else {
+ SDValue Ops[] = {N0, OverflowVal};
+ N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops);
+ }
}
}
+ C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
}
if (!N0)
N0 = DAG.getConstant(0, DL, MVT::i32);
if (!C1)
C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32));
- return {N0, SDValue(C1, 0)};
+ return {N0, SOffset, SDValue(C1, 0)};
}
// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c98426cdac0b1..073d577eaad63 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -18,6 +18,7 @@
#include "AMDGPUISelLowering.h"
#include "SIDefines.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include <tuple>
namespace llvm {
@@ -104,14 +105,14 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
- // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
- // (the offset that is included in bounds checking and swizzling, to be split
- // between the instruction's voffset and immoffset fields) and soffset (the
- // offset that is excluded from bounds checking and swizzling, to go in the
- // instruction's soffset field). This function takes the first kind of
- // offset and figures out how to split it between voffset and immoffset.
- std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
- SelectionDAG &DAG) const;
+ // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
+ // offset (included in bounds checking and swizzling) and soffset (excluded
+ // from bounds checking and swizzling, but affecting raw.* num_records
+ // checking). This splits them into the instruction's voffset, soffset, and
+ // immoffset fields.
+ std::tuple<SDValue, SDValue, SDValue>
+ splitBufferOffsets(SDValue Offset, SDValue SOffset, SelectionDAG &DAG,
+ bool HasLinearOOB) const;
SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 8ec540029e2a0..e4caec9016d36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1221,12 +1221,10 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
@@ -1239,9 +1237,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 58720256
- ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
index 55561c489402a..aeca1ebc274df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
@@ -61,27 +61,26 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16(ptr addrspace(8) inreg %rsrc, i32 %voffset.base) {
; GFX8-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
; GFX8-NEXT: s_mov_b32 s0, s2
; GFX8-NEXT: s_mov_b32 s1, s3
; GFX8-NEXT: s_mov_b32 s2, s4
; GFX8-NEXT: s_mov_b32 s3, s5
-; GFX8-NEXT: v_or_b32_e32 v0, 0x3fffffc, v0
-; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 16 offen
+; GFX8-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
+; GFX8-NEXT: s_mov_b32 s4, 0x3fff010
+; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], s4 offen offset:4092
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0x3fffffc
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xfc000000, v0
; GFX1250-NEXT: s_mov_b32 s0, s2
; GFX1250-NEXT: s_mov_b32 s1, s3
; GFX1250-NEXT: s_mov_b32 s2, s4
; GFX1250-NEXT: s_mov_b32 s3, s5
-; GFX1250-NEXT: v_and_or_b32 v0, 0xfc000000, v0, v1
-; GFX1250-NEXT: s_mov_b32 s4, 16
-; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen
+; GFX1250-NEXT: s_mov_b32 s4, 0x3800010
+; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
%voffset.masked = and i32 %voffset.base, -67108864
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 952b7f33d3dfa..97ae0a66aedc6 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -2652,8 +2652,7 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX908-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
; GFX908-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
; GFX908-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
- ; GFX908-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 67104768, implicit $exec
+ ; GFX908-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 67104768
; GFX908-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.1:
@@ -2674,13 +2673,13 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX908-NEXT: bb.2:
; GFX908-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX908-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 4092, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; GFX908-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; GFX908-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3:
; GFX908-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
- ; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+ ; GFX908-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
; GFX908-NEXT: SI_RETURN implicit $vgpr0
;
; GFX1250-LABEL: name: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0
@@ -2699,14 +2698,14 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[COPY5]], %subreg.sub2, killed [[COPY4]], %subreg.sub3
- ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 58720256, implicit $exec
- ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 58720256
; GFX1250-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX1250-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.1:
; GFX1250-NEXT: successors: %bb.2(0x80000000)
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %30, %bb.2
+ ; GFX1250-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_2]], %bb.0, %30, %bb.2
; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub0, implicit $exec
; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE2]].sub1, implicit $exec
; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
@@ -2720,13 +2719,13 @@ define float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset0(ptr a
; GFX1250-NEXT: bb.2:
; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], $sgpr_null, 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
+ ; GFX1250-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET killed [[REG_SEQUENCE5]], [[S_MOV_B32_]], 8388604, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.rsrc, align 1, addrspace 8)
; GFX1250-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX1250-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.3:
- ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET]]
; GFX1250-NEXT: SI_RETURN implicit $vgpr0
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 0, i32 0)
ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index 68613502b43be..97c4e74eaf734 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10,SI
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10,VI
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
@@ -247,24 +247,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -279,24 +286,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -311,24 +325,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -343,31 +364,38 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: buffer_load_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX12-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8388604
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@@ -435,6 +463,20 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) {
+; SI-LABEL: buffer_load_negative_offset:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_negative_offset:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_negative_offset:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0
@@ -462,6 +504,30 @@ main_body:
}
define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, ptr addrspace(3) %lds) {
+; SI-LABEL: buffer_load_mmo:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 16, v0
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, v1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_mmo:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_write2_b32 v0, v2, v2 offset1:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, v1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_mmo:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0
@@ -596,39 +662,17 @@ define amdgpu_ps void @buffer_load_x1_offen_merged_or(<4 x i32> inreg %rsrc, i32
; GFX11-NEXT: exp mrt0, v4, v5, v0, v0 done
; GFX11-NEXT: s_endpgm
;
-; GFX12-SDAG-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-SDAG: ; %bb.0: ; %main_body
-; GFX12-SDAG-NEXT: v_lshlrev_b32_e32 v4, 6, v0
-; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
-; GFX12-SDAG-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX12-SDAG-NEXT: export mrt0, v0, v1, v2, v3 done
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: export mrt0, v4, v5, v0, v0 done
-; GFX12-SDAG-NEXT: s_endpgm
-;
-; GFX12-GISEL-LABEL: buffer_load_x1_offen_merged_or:
-; GFX12-GISEL: ; %bb.0: ; %main_body
-; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v0, 6, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, 4, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, 8, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, 12, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v4, 16, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v5, 28, v0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, 32, v0
-; GFX12-GISEL-NEXT: s_clause 0x5
-; GFX12-GISEL-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v2, v2, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v3, v3, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v4, v4, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v5, v5, s[0:3], null offen
-; GFX12-GISEL-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x2
-; GFX12-GISEL-NEXT: export mrt0, v1, v2, v3, v4 done
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: export mrt0, v5, v0, v0, v0 done
-; GFX12-GISEL-NEXT: s_endpgm
+; GFX12-LABEL: buffer_load_x1_offen_merged_or:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 6, v0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:4
+; GFX12-NEXT: buffer_load_b64 v[4:5], v4, s[0:3], null offen offset:28
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: export mrt0, v0, v1, v2, v3 done
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: export mrt0, v4, v5, v0, v0 done
+; GFX12-NEXT: s_endpgm
main_body:
%a = shl i32 %inp, 6
%a1 = or i32 %a, 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
index 12997dfa5fabe..94e042104dbbf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -1,11 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck %s --check-prefixes=PREGFX10
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefixes=PREGFX10
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=PREGFX10,SI
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=PREGFX10,VI
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefixes=GFX10
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX1250
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=SI
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=VI
define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) {
; PREGFX10-LABEL: buffer_load:
@@ -307,24 +305,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -340,24 +345,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -373,24 +385,31 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -406,32 +425,39 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: buffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: buffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: buffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: buffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], 0 offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: buffer_load_voffset_large_24bit:
; GFX1250: ; %bb.0: ; %main_body
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
+; GFX1250-NEXT: s_mov_b32 s4, 0x800000
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
main_body:
@@ -439,7 +465,6 @@ main_body:
ret <4 x float> %data
}
-
define amdgpu_ps float @buffer_load_x1(ptr addrspace(8) inreg %rsrc, i32 %ofs) {
; PREGFX10-LABEL: buffer_load_x1:
; PREGFX10: ; %bb.0: ; %main_body
@@ -501,6 +526,20 @@ main_body:
}
define amdgpu_ps <4 x float> @buffer_load_negative_offset(ptr addrspace(8) inreg, i32 %ofs) {
+; SI-LABEL: buffer_load_negative_offset:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_negative_offset:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_negative_offset:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0
@@ -529,6 +568,30 @@ main_body:
}
define amdgpu_ps float @buffer_load_mmo(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %lds) {
+; SI-LABEL: buffer_load_mmo:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v2, 0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: v_add_i32_e32 v0, vcc, 16, v0
+; SI-NEXT: ds_write_b32 v0, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, v1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: buffer_load_mmo:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v2, 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_write2_b32 v0, v2, v2 offset1:4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, v1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: buffer_load_mmo:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0
@@ -875,6 +938,24 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, p
; }
define amdgpu_ps void @raw_ptr_buffer_load_v8f16(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %ptr) {
+; SI-LABEL: raw_ptr_buffer_load_v8f16:
+; SI: ; %bb.0:
+; SI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; SI-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ds_write_b64 v5, v[3:4]
+; SI-NEXT: ds_write_b64 v0, v[1:2]
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: raw_ptr_buffer_load_v8f16:
+; VI: ; %bb.0:
+; VI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ds_write_b128 v0, v[1:4]
+; VI-NEXT: s_endpgm
+;
; GFX10-LABEL: raw_ptr_buffer_load_v8f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
@@ -980,6 +1061,24 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4i16(ptr addrspace(8) inreg %rsrc, p
; }
define amdgpu_ps void @raw_ptr_buffer_load_v8i16(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) %ptr) {
+; SI-LABEL: raw_ptr_buffer_load_v8i16:
+; SI: ; %bb.0:
+; SI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; SI-NEXT: v_add_i32_e32 v5, vcc, 8, v0
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ds_write_b64 v5, v[3:4]
+; SI-NEXT: ds_write_b64 v0, v[1:2]
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: raw_ptr_buffer_load_v8i16:
+; VI: ; %bb.0:
+; VI-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ds_write_b128 v0, v[1:4]
+; VI-NEXT: s_endpgm
+;
; GFX10-LABEL: raw_ptr_buffer_load_v8i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: buffer_load_dwordx4 v[1:4], off, s[0:3], 0
@@ -1497,6 +1596,20 @@ define <2 x ptr addrspace(2)> @buffer_load_v2p2__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(2)> @buffer_load_v3p2__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p2__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p2__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p2__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1630,6 +1743,20 @@ define <2 x ptr addrspace(3)> @buffer_load_v2p3__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(3)> @buffer_load_v3p3__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p3__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p3__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p3__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1763,6 +1890,20 @@ define <2 x ptr addrspace(5)> @buffer_load_v2p5__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(5)> @buffer_load_v3p5__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p5__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p5__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p5__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1896,6 +2037,20 @@ define <2 x ptr addrspace(6)> @buffer_load_v2p6__voffset_add(ptr addrspace(8) in
}
define <3 x ptr addrspace(6)> @buffer_load_v3p6__voffset_add(ptr addrspace(8) inreg %rsrc, i32 %voffset) {
+; SI-LABEL: buffer_load_v3p6__voffset_add:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[16:19], 0 offen offset:60
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: buffer_load_v3p6__voffset_add:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: buffer_load_dwordx3 v[0:2], v0, s[16:19], 0 offen offset:60
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
; GFX10-LABEL: buffer_load_v3p6__voffset_add:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1960,12 +2115,39 @@ define <4 x ptr addrspace(6)> @buffer_load_v4p6__voffset_add(ptr addrspace(8) in
define amdgpu_ps <4 x float> @raw_ptr_large_zero_soffset(ptr addrspace(8) inreg %rsrc) {
; SI-LABEL: raw_ptr_large_zero_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
;
; VI-LABEL: raw_ptr_large_zero_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 0 offen offset:4092
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: raw_ptr_large_zero_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_zero_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_zero_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 0, i32 0)
ret <4 x float> %data
@@ -1973,25 +2155,74 @@ main_body:
define amdgpu_ps <4 x float> @raw_ptr_large_safe_soffset(ptr addrspace(8) inreg %rsrc) {
; SI-LABEL: raw_ptr_large_safe_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 16 offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
;
; VI-LABEL: raw_ptr_large_safe_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], 16 offen offset:4092
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1010
+; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: raw_ptr_large_safe_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: s_movk_i32 s4, 0x1010
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], s4 offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_safe_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_movk_i32 s4, 0x1010
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_safe_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s4, 16
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 16, i32 0)
ret <4 x float> %data
}
define amdgpu_ps <4 x float> @raw_ptr_large_overflowing_soffset(ptr addrspace(8) inreg %rsrc) {
-; SI-LABEL: raw_ptr_large_overflowing_soffset:
-; SI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; SI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+; PREGFX10-LABEL: raw_ptr_large_overflowing_soffset:
+; PREGFX10: ; %bb.0: ; %main_body
+; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
+; PREGFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], -16 offen offset:4092
+; PREGFX10-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-NEXT: ; return to shader part epilog
;
-; VI-LABEL: raw_ptr_large_overflowing_soffset:
-; VI: v_mov_b32_e32 [[VOFF:v[0-9]+]], 0x1000
-; VI-NEXT: buffer_load_dwordx4 v[0:3], [[VOFF]], s[0:3], -16 offen offset:4092
+; GFX10-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX10: ; %bb.0: ; %main_body
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX10-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], -16 offen offset:4092
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
+; GFX11-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], -16 offen offset:4092
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: raw_ptr_large_overflowing_soffset:
+; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b32 s4, -16
+; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:8188
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
main_body:
%data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %rsrc, i32 8188, i32 -16, i32 0)
ret <4 x float> %data
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
index b3115253bf29f..95b3276a8e5aa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
@@ -96,24 +96,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_13bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -122,24 +129,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_16bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -148,24 +162,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_23bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
@@ -174,24 +195,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_24bit(ptr addrspace(8) inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
index f778304c414a9..eda2c0247bcc1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefixes=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefixes=GFX11 %s
@@ -166,22 +166,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_13bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x1000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 8188, i32 0, i32 63, i32 0)
@@ -189,22 +195,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_16bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xf000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 65532, i32 0, i32 63, i32 0)
@@ -212,22 +224,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_23bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x7ff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 8388604, i32 0, i32 63, i32 0)
@@ -235,22 +253,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_24bit(ptr addrspace(8) inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xfff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f32(<4 x float> %data, ptr addrspace(8) %rsrc, i32 16777212, i32 0, i32 63, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
index f01e85a2e4a02..7626e58e9b506 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck -check-prefix=GFX12 %s
@@ -120,24 +120,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_13bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x1000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x1000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -152,24 +159,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_16bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xf000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xf000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -184,24 +198,31 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_23bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0x7ff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0x7ff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
@@ -216,31 +237,38 @@ main_body:
}
define amdgpu_ps <4 x float> @tbuffer_load_voffset_large_24bit(<4 x i32> inreg) {
-; PREGFX10-LABEL: tbuffer_load_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; PREGFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_waitcnt vmcnt(0)
-; PREGFX10-NEXT: ; return to shader part epilog
+; SI-LABEL: tbuffer_load_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v0, 0xfff000
+; SI-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ; return to shader part epilog
+;
+; VI-LABEL: tbuffer_load_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: tbuffer_load_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: tbuffer_load_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xfff000
-; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: tbuffer_load_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX12-NEXT: tbuffer_load_format_xyzw v[0:3], v0, s[0:3], null format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: tbuffer_load_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388604
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
index 8afa43a6ebe48..888d91e6a5691 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefix=PREGFX10 %s
-;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=PREGFX10 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=verde | FileCheck -check-prefixes=PREGFX10,SI %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=PREGFX10,VI %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefix=GFX11 %s
;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck -check-prefix=GFX12 %s
@@ -207,22 +207,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_13bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_13bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_13bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x1000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_13bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_movk_i32 s4, 0x1000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_13bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_movk_i32 s4, 0x1000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_13bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x1000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_movk_i32 s4, 0x1000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_13bit:
@@ -235,22 +241,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_16bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_16bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_16bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xf000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_16bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xf000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_16bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xf000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_16bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xf000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xf000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_16bit:
@@ -263,22 +275,28 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_23bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_23bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_23bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0x7ff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_23bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0x7ff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_23bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_23bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0x7ff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0x7ff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_23bit:
@@ -291,28 +309,34 @@ main_body:
}
define amdgpu_ps void @buffer_store_voffset_large_24bit(<4 x i32> inreg %rsrc, <4 x float> %data) {
-; PREGFX10-LABEL: buffer_store_voffset_large_24bit:
-; PREGFX10: ; %bb.0: ; %main_body
-; PREGFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; PREGFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
-; PREGFX10-NEXT: s_endpgm
+; SI-LABEL: buffer_store_voffset_large_24bit:
+; SI: ; %bb.0: ; %main_body
+; SI-NEXT: v_mov_b32_e32 v4, 0xfff000
+; SI-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offen offset:4092
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: buffer_store_voffset_large_24bit:
+; VI: ; %bb.0: ; %main_body
+; VI-NEXT: s_mov_b32 s4, 0xfff000
+; VI-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_DATA_FORMAT_RESERVED_15,BUF_NUM_FORMAT_SSCALED] offset:4092
+; VI-NEXT: s_endpgm
;
; GFX10-LABEL: buffer_store_voffset_large_24bit:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_SINT] offen offset:4092
+; GFX10-NEXT: s_mov_b32 s4, 0xfff000
+; GFX10-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_SINT] offset:4092
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: buffer_store_voffset_large_24bit:
; GFX11: ; %bb.0: ; %main_body
-; GFX11-NEXT: v_mov_b32_e32 v4, 0xfff000
-; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:4092
+; GFX11-NEXT: s_mov_b32 s4, 0xfff000
+; GFX11-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:4092
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: buffer_store_voffset_large_24bit:
; GFX12: ; %bb.0: ; %main_body
-; GFX12-NEXT: v_mov_b32_e32 v4, 0x800000
-; GFX12-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], null format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:8388604
+; GFX12-NEXT: s_mov_b32 s4, 0x800000
+; GFX12-NEXT: tbuffer_store_format_xyzw v[0:3], off, s[0:3], s4 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388604
; GFX12-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %data, <4 x i32> %rsrc, i32 16777212, i32 0, i32 63, i32 0)
>From 7f383aaf0a8061f52d3142c2b2ab4d497564351e Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Mon, 6 Jul 2026 22:09:19 +0000
Subject: [PATCH 2/3] Test update failure, whoops
---
.../llvm.amdgcn.raw.ptr.buffer.offset-split.ll | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
index aeca1ebc274df..a84f56dd91e94 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
@@ -9,21 +9,20 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
; GFX8-NEXT: s_mov_b32 s1, s3
; GFX8-NEXT: s_mov_b32 s2, s4
; GFX8-NEXT: s_mov_b32 s3, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x3fff000
-; GFX8-NEXT: buffer_load_dword v0, v0, s[0:3], 16 offen offset:4092
+; GFX8-NEXT: s_mov_b32 s4, 0x3fff010
+; GFX8-NEXT: buffer_load_dword v0, off, s[0:3], s4 offset:4092
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0x3800000
; GFX1250-NEXT: s_mov_b32 s0, s2
; GFX1250-NEXT: s_mov_b32 s1, s3
; GFX1250-NEXT: s_mov_b32 s2, s4
; GFX1250-NEXT: s_mov_b32 s3, s5
-; GFX1250-NEXT: s_mov_b32 s4, 16
-; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], s4 offen offset:8388604
+; GFX1250-NEXT: s_mov_b32 s4, 0x3800010
+; GFX1250-NEXT: buffer_load_b32 v0, off, s[0:3], s4 offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
%val = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %rsrc, i32 67108860, i32 16, i32 0)
>From b25a7095e4b4fc735832e35b049af50f337d9247 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Fri, 10 Jul 2026 19:36:59 +0000
Subject: [PATCH 3/3] Review feedback
---
.../Target/AMDGPU/AMDGPUGlobalISelUtils.cpp | 1 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 29 +++++++------------
2 files changed, 11 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index 2bfeb0546e1af..889547fece3f8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -46,6 +46,7 @@ AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
return std::pair(Reg, 0);
}
+ // TODO: Handle G_OR used for add case
if (mi_match(Def->getOperand(2).getReg(), MRI, m_ICst(Offset)))
return std::pair(Def->getOperand(1).getReg(), Offset);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 7281bec93850b..2b459432f22ff 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6507,11 +6507,10 @@ static bool canMoveBufferOffsetToSOffset(const GCNSubtarget &ST,
static std::optional<uint32_t>
getCombinedBufferSOffset(MachineRegisterInfo &MRI, Register SOffset,
uint32_t Offset) {
- APInt C;
+ int64_t C;
if (!mi_match(SOffset, MRI, m_ICst(C)))
return std::nullopt;
- return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C.getZExtValue()),
- Offset);
+ return checkedAddUnsigned<uint32_t>(static_cast<uint32_t>(C), Offset);
}
// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
@@ -6536,22 +6535,14 @@ AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no unsigned
// overflow.
- bool CheckNUW = ST.hasGFX1250Insts();
- std::tie(BaseReg, ImmOffset) =
- AMDGPU::getBaseWithConstantOffset(MRI, OrigOffset, VT, CheckNUW);
- bool IsNUW = false;
- if (ImmOffset) {
- if (!BaseReg) {
- IsNUW = true;
- } else if (MachineInstr *OffsetDef =
- getDefIgnoringCopies(OrigOffset, MRI)) {
- // Match SelectionDAG: a G_OR with a constant reaches here only when it is
- // add-like, and such an OR satisfies the no-wrap condition.
- IsNUW = OffsetDef->getOpcode() == TargetOpcode::G_OR ||
- (OffsetDef->getOpcode() == TargetOpcode::G_ADD &&
- OffsetDef->getFlag(MachineInstr::NoUWrap));
- }
- }
+ std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
+ MRI, OrigOffset, VT, /*CheckNUW=*/ST.hasGFX1250Insts());
+ // Otherwise, we don't check for nuw when getting the constant offset, but we
+ // do need to re-check adds because only nuw adds can be moved to soffset.
+ bool IsNUW = true;
+ if (MachineInstr *OffsetDef = getDefIgnoringCopies(OrigOffset, MRI))
+ IsNUW = OffsetDef->getOpcode() != TargetOpcode::G_ADD ||
+ !OffsetDef->getFlag(MachineInstr::NoUWrap);
// If BaseReg is a pointer, convert it to int.
if (BaseReg && MRI.getType(BaseReg).isPointer())
More information about the llvm-commits
mailing list