[llvm] [AMDGPU] Support TFE D16 format buffer loads on gfx8/10/11/12 (PR #211465)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 11:27:26 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/211465
>From 2b514271414d3a0f5642705eb1cca1a633ba6b72 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 23 Jul 2026 08:00:25 +0200
Subject: [PATCH 1/7] [AMDGPU] Reject unsupported TFE D16 format buffer loads
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 10 ++++++++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 23 ++++++++++++++-----
...mdgcn.struct.ptr.buffer.format.i8.xfail.ll | 23 +++++++++++++++++++
3 files changed, 50 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 87e85bef7c701..8db7fd4d5541d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6901,6 +6901,16 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
+ if (IsFormat && !IsTyped && IsD16 && IsTFE) {
+ const Function &Fn = B.getMF().getFunction();
+ Fn.getContext().diagnose(DiagnosticInfoUnsupported(
+ Fn, "unsupported TFE D16 format buffer load", MI.getDebugLoc()));
+ B.buildUndef(Dst);
+ B.buildUndef(StatusDst);
+ MI.eraseFromParent();
+ return true;
+ }
+
std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
unsigned Opc;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8156de7a24b01..b2ce3337bac02 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7835,6 +7835,14 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
+ if (IsD16 && IsTFE) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported TFE D16 format buffer load", DL.getDebugLoc()));
+ return DAG.getMergeValues(
+ {DAG.getPOISON(LoadVT), DAG.getPOISON(MVT::i32), M->getOperand(0)}, DL);
+ }
+
unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
: AMDGPUISD::BUFFER_LOAD_FORMAT)
: IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
@@ -7855,12 +7863,15 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
}
EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
- SDVTList VTList = DAG.getVTList(CastVT, MVT::Other);
+ SDVTList VTList = IsTFE ? DAG.getVTList(CastVT, MVT::i32, MVT::Other)
+ : DAG.getVTList(CastVT, MVT::Other);
SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
M->getMemOperand(), DAG);
- return DAG.getMergeValues(
- {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)},
- DL);
+ SDValue Data = DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode);
+ if (IsTFE)
+ return DAG.getMergeValues({Data, MemNode.getValue(1), MemNode.getValue(2)},
+ DL);
+ return DAG.getMergeValues({Data, MemNode.getValue(1)}, DL);
}
static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
@@ -8407,8 +8418,8 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(Res.getOperand(I));
}
} else {
- Results.push_back(Res);
- Results.push_back(Res.getValue(1));
+ for (unsigned I = 0; I < N->getNumValues(); ++I)
+ Results.push_back(Res.getValue(I));
}
return;
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
index f5f8471f4dc1f..df137c5df98ea 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
@@ -18,3 +18,26 @@ define amdgpu_ps void @store_i8(ptr addrspace(8) inreg %rsrc, i8 %data, i32 %ind
call void @llvm.amdgcn.struct.ptr.buffer.store.format.i8(i8 %data, ptr addrspace(8) %rsrc, i32 %index, i32 0, i32 0, i32 0)
ret void
}
+
+; A D16 buffer.load.format combined with TFE has no corresponding real
+; instruction (no TFE variant of the D16 format load opcodes exists in
+; hardware), so it must be refused instead of miscounting SDNode results.
+; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+ %r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<3 x i16>, i32} %r, 0
+ %st = extractvalue {<3 x i16>, i32} %r, 1
+ store <3 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+ %r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {half, i32} %r, 0
+ %st = extractvalue {half, i32} %r, 1
+ store half %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
>From e8d9ae05ac893900d832624e41173e8fbbdc2800 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 30 Jul 2026 11:40:46 +0200
Subject: [PATCH 2/7] Address comment
---
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 1 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 63 ++++-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 8 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 2 +
llvm/lib/Target/AMDGPU/BUFInstructions.td | 6 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 46 ++-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 3 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 1 +
...mdgcn.struct.ptr.buffer.format.i8.xfail.ll | 16 +-
...n.struct.ptr.buffer.load.format.d16.tfe.ll | 261 ++++++++++++++++++
11 files changed, 395 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 5df9834f4ef80..df4a197009bf6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -296,6 +296,7 @@ def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SBYTE_TFE, SIbuffer_load_byte_tfe>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT, SIbuffer_load_format>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_TFE, SIbuffer_load_format_tfe>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16, SIbuffer_load_format_d16>;
+def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE, SIbuffer_load_format_d16_tfe>;
def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT, SItbuffer_load>;
def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT_D16, SItbuffer_load_d16>;
def : GINodeEquiv<G_AMDGPU_BUFFER_STORE, SIbuffer_store>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c5cea131dc64b..3205213c86f04 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6862,10 +6862,11 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
- if (IsFormat && !IsTyped && IsD16 && IsTFE) {
+ if (IsFormat && !IsTyped && IsD16 && IsTFE && ST.hasGFX90AInsts()) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
- Fn, "unsupported TFE D16 format buffer load", MI.getDebugLoc()));
+ Fn, "TFE D16 format buffer load is not supported on this GPU",
+ MI.getDebugLoc()));
B.buildUndef(Dst);
B.buildUndef(StatusDst);
MI.eraseFromParent();
@@ -6884,9 +6885,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
} else if (IsFormat) {
if (IsD16) {
- if (IsTFE)
- return false;
- Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
+ Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE
+ : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
} else {
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
@@ -6908,7 +6908,58 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
}
}
- if (IsTFE) {
+ if (IsTFE && IsD16 && Ty.isVector()) {
+ // D16 dwords are packed (or, on unpacked-D16 targets, one element per
+ // dword) at a different granularity than Ty's own bit width, so the
+ // dwords loaded for the value do not necessarily cover Ty exactly (e.g.
+ // v3i16 needs 2 dwords = 64 bits for its 48 bits of data). Load the
+ // dwords, then repack/truncate down to Ty the same way the MUBUF/MIMG
+ // TFE v3i16 case does.
+ const unsigned NumElts = Ty.getNumElements();
+ const unsigned NumValueDWords =
+ Unpacked ? NumElts : divideCeil(NumElts * 16, 32);
+ const unsigned NumLoadDWords = NumValueDWords + 1;
+ LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
+ Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
+ buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
+ Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
+
+ SmallVector<Register, 5> LoadElts;
+ for (unsigned I = 0; I != NumValueDWords; ++I)
+ LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(I32));
+ LoadElts.push_back(StatusDst);
+ B.buildUnmerge(LoadElts, LoadDstReg);
+ LoadElts.truncate(NumValueDWords);
+
+ Register PackedI16;
+ if (Unpacked) {
+ SmallVector<Register, 4> Repack;
+ for (Register R : LoadElts)
+ Repack.push_back(B.buildTrunc(LLT::integer(16), R).getReg(0));
+ LLT RepackedTy = LLT::fixed_vector(NumValueDWords, LLT::integer(16));
+ PackedI16 = B.buildMergeLikeInstr(RepackedTy, Repack).getReg(0);
+ } else {
+ LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
+ Register Merged =
+ NumValueDWords == 1
+ ? LoadElts[0]
+ : B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ LLT PackedI16Ty = LLT::fixed_vector(NumValueDWords * 2, LLT::integer(16));
+ PackedI16 = B.buildBitcast(PackedI16Ty, Merged).getReg(0);
+ }
+
+ LLT PackedI16Ty = B.getMRI()->getType(PackedI16);
+ LLT DstIntTy = Ty.changeElementType(LLT::integer(16));
+ Register ValueI16 = PackedI16;
+ if (PackedI16Ty.getNumElements() != NumElts) {
+ ValueI16 = B.getMRI()->createGenericVirtualRegister(DstIntTy);
+ B.buildDeleteTrailingVectorElements(ValueI16, PackedI16);
+ }
+ if (EltTy.isFloat())
+ B.buildBitcast(Dst, ValueI16);
+ else
+ B.buildCopy(Dst, ValueI16);
+ } else if (IsTFE) {
unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
unsigned NumLoadDWords = NumValueDWords + 1;
LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index b9bc59a79a642..827e491b97abc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1296,6 +1296,14 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+ addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE}, StandardB)
+ .Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+
addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
// waterfall expansion is part of S_BUF_to_BUF
.Any({{UniB32}, {{SgprB32}, {SgprV4S32, Sgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 21472bb886196..a4edd4b699f27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3100,6 +3100,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
+ case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
case AMDGPU::G_AMDGPU_BUFFER_STORE:
@@ -4499,6 +4500,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
+ case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 5faa478407c34..f20648df617e4 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1468,6 +1468,9 @@ let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
+
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80_TFE">;
} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
@@ -1486,6 +1489,9 @@ let True16Predicate = UseRealTrue16Insts in {
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
+
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE">;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE">;
} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
foreach vt = Reg32Types.types in {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6f52f7c098b97..c8bfc808cc2f3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7792,7 +7792,8 @@ static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
SelectionDAG &DAG,
ArrayRef<SDValue> Ops,
- bool IsIntrinsic) const {
+ bool IsIntrinsic,
+ bool IsTFE) const {
SDLoc DL(M);
bool Unpacked = Subtarget->hasUnpackedD16VMem();
@@ -7811,6 +7812,40 @@ SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
}
}
+ if (IsTFE) {
+ // The hardware always returns TFE results at dword granularity: the data
+ // dwords followed by one status dword. Load that combined vector, then
+ // split it into the status and the D16 data before packing/truncating
+ // the data the same way as the non-TFE case.
+ unsigned NumValueDWords = divideCeil(EquivLoadVT.getSizeInBits(), 32);
+ unsigned NumLoadDWords = NumValueDWords + 1;
+ EVT LoadVTList =
+ EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumLoadDWords);
+ SDVTList VTList = DAG.getVTList(LoadVTList, MVT::Other);
+ SDValue Load = DAG.getMemIntrinsicNode(
+ Opcode, DL, VTList, Ops, M->getMemoryVT(), M->getMemOperand());
+ SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load,
+ DAG.getVectorIdxConstant(NumValueDWords, DL));
+ SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
+ SDValue ValueDWords =
+ NumValueDWords == 1
+ ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load, ZeroIdx)
+ : DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL,
+ EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumValueDWords),
+ Load, ZeroIdx);
+ // A scalar D16 result (f16/i16) occupies less than a full dword, so
+ // truncate before bitcasting to the final scalar type.
+ if (!EquivLoadVT.isVector() && EquivLoadVT.getSizeInBits() < 32)
+ ValueDWords = DAG.getNode(ISD::TRUNCATE, DL,
+ EquivLoadVT.changeTypeToInteger(), ValueDWords);
+ SDValue Value = DAG.getNode(ISD::BITCAST, DL, EquivLoadVT, ValueDWords);
+ SDValue Adjusted =
+ adjustLoadValueTypeImpl(Value, LoadVT, DL, DAG, Unpacked);
+ return DAG.getMergeValues({Adjusted, Status, SDValue(Load.getNode(), 1)},
+ DL);
+ }
+
// Change from v4f16/v2f16 to EquivLoadVT.
SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
@@ -7843,10 +7878,11 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
- if (IsD16 && IsTFE) {
+ if (IsD16 && IsTFE && Subtarget->hasGFX90AInsts()) {
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
- "unsupported TFE D16 format buffer load", DL.getDebugLoc()));
+ "TFE D16 format buffer load is not supported on this GPU",
+ DL.getDebugLoc()));
return DAG.getMergeValues(
{DAG.getPOISON(LoadVT), DAG.getPOISON(MVT::i32), M->getOperand(0)}, DL);
}
@@ -7857,7 +7893,9 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
: AMDGPUISD::BUFFER_LOAD;
if (IsD16) {
- return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
+ return adjustLoadValueType(IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
+ : AMDGPUISD::BUFFER_LOAD_FORMAT_D16,
+ M, DAG, Ops, /*IsIntrinsic=*/false, IsTFE);
}
// Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 86653fe6920c5..e56aaf37270d2 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -133,9 +133,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
- SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
- SelectionDAG &DAG, ArrayRef<SDValue> Ops,
- bool IsIntrinsic = false) const;
+ SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M, SelectionDAG &DAG,
+ ArrayRef<SDValue> Ops, bool IsIntrinsic = false,
+ bool IsTFE = false) const;
SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
ArrayRef<SDValue> Ops) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 8208a6ea2d5b5..066bfe564969c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -172,6 +172,9 @@ def SIbuffer_load_format_tfe : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_TFE", SDTB
def SIbuffer_load_format_d16 : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_D16",
SDTBufferLoad,
[SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
+def SIbuffer_load_format_d16_tfe : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE",
+ SDTBufferLoad,
+ [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
def SDTBufferStore : SDTypeProfile<0, 8,
[ // vdata
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f04373d3163a9..9ee8af5cd3b98 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -4775,6 +4775,7 @@ def G_AMDGPU_BUFFER_LOAD_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction;
+def G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction;
def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
index df137c5df98ea..9f5efef3796f2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
@@ -1,11 +1,14 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=0 -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
+; RUN: not llc -global-isel -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
; An i8 buffer.load.format / buffer.store.format has no corresponding real
; instruction (no byte-granularity format access exists in hardware), so both
; SelectionDAG and GlobalISel must refuse to lower it.
; CHECK: error: {{.*}}unsupported sub-dword format buffer load
+; GFX90A: error: {{.*}}unsupported sub-dword format buffer load
define amdgpu_ps float @load_i8(ptr addrspace(8) inreg %rsrc) {
%data = call i8 @llvm.amdgcn.struct.ptr.buffer.load.format.i8(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%zext = zext i8 %data to i32
@@ -14,15 +17,18 @@ define amdgpu_ps float @load_i8(ptr addrspace(8) inreg %rsrc) {
}
; CHECK: error: {{.*}}unsupported sub-dword format buffer store
+; GFX90A: error: {{.*}}unsupported sub-dword format buffer store
define amdgpu_ps void @store_i8(ptr addrspace(8) inreg %rsrc, i8 %data, i32 %index) {
call void @llvm.amdgcn.struct.ptr.buffer.store.format.i8(i8 %data, ptr addrspace(8) %rsrc, i32 %index, i32 0, i32 0, i32 0)
ret void
}
-; A D16 buffer.load.format combined with TFE has no corresponding real
-; instruction (no TFE variant of the D16 format load opcodes exists in
-; hardware), so it must be refused instead of miscounting SDNode results.
-; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+; D16 buffer.load.format combined with TFE has no real hardware encoding on
+; gfx90a, so it must be refused there. Other targets (gfx8/gfx10/gfx11/gfx12)
+; have real TFE encodings and are covered by
+; llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll instead.
+; CHECK-NOT: error: {{.*}}TFE D16 format buffer load
+; GFX90A: error: {{.*}}TFE D16 format buffer load is not supported on this GPU
define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
%r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue {<3 x i16>, i32} %r, 0
@@ -32,7 +38,7 @@ define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
ret void
}
-; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+; GFX90A: error: {{.*}}TFE D16 format buffer load is not supported on this GPU
define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
%r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue {half, i32} %r, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
new file mode 100644
index 0000000000000..2499182123c6b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-UNPACKED-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-UNPACKED-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu8.10 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-PACKED-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-PACKED-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX9-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX9-GISEL %s
+
+define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_x v[0:1], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_short v[2:3], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_x v[0:1], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_x v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_short v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_x v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_x v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_x v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {half, i32} %r, 0
+ %st = extractvalue {half, i32} %r, 1
+ store half %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v3i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyz v[0:3], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: s_add_u32 s0, s4, 4
+; GFX8-UNPACKED-SDAG-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v9, s1
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_short v[8:9], v2
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[6:7], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[4:5], v3
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v3i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyz v[0:3], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: s_add_u32 s0, s4, 2
+; GFX8-UNPACKED-GISEL-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v9, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s0
+; GFX8-UNPACKED-GISEL-NEXT: s_add_u32 s0, s4, 4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-UNPACKED-GISEL-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v11, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v10, s0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[4:5], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[8:9], v1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[10:11], v2
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[6:7], v3
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v3i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyz v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: s_add_u32 s0, s12, 4
+; GFX8-PACKED-SDAG-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v8, s1
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v7, s0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s15
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s13
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_short v[7:8], v1
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[3:4], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v3i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyz v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-PACKED-GISEL-NEXT: s_add_u32 s0, s12, 2
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-PACKED-GISEL-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-PACKED-GISEL-NEXT: s_add_u32 s0, s12, 4
+; GFX8-PACKED-GISEL-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v3i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyz v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_short v0, v2, s[12:13] offset:4
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v3i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyz v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_short v0, v2, s[12:13] offset:4
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13] offset:2
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<3 x i16>, i32} %r, 0
+ %st = extractvalue {<3 x i16>, i32} %r, 1
+ store <3 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+declare {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8), i32, i32, i32, i32)
>From be54b711b15a042a8b9ea08c1b551b010a307027 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 4 Aug 2026 17:42:12 +0200
Subject: [PATCH 3/7] Address comments
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 12 +-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 5 +-
llvm/lib/Target/AMDGPU/BUFInstructions.td | 12 +-
...n.struct.ptr.buffer.load.format.d16.tfe.ll | 502 ++++++++++++++++++
4 files changed, 523 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 3205213c86f04..d05e9f0dfd98b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6939,11 +6939,13 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
LLT RepackedTy = LLT::fixed_vector(NumValueDWords, LLT::integer(16));
PackedI16 = B.buildMergeLikeInstr(RepackedTy, Repack).getReg(0);
} else {
- LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
- Register Merged =
- NumValueDWords == 1
- ? LoadElts[0]
- : B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ Register Merged;
+ if (NumValueDWords == 1) {
+ Merged = LoadElts[0];
+ } else {
+ LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
+ Merged = B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ }
LLT PackedI16Ty = LLT::fixed_vector(NumValueDWords * 2, LLT::integer(16));
PackedI16 = B.buildBitcast(PackedI16Ty, Merged).getReg(0);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 827e491b97abc..2cc0a55cb3368 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1302,7 +1302,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+ .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Any({{DivB160}, {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
+ .Any({{UniB160},
+ {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
// waterfall expansion is part of S_BUF_to_BUF
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index f20648df617e4..ffc31aedc480e 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1470,7 +1470,9 @@ let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80_TFE">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v5i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_TFE">;
} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
@@ -1490,8 +1492,14 @@ let True16Predicate = UseRealTrue16Insts in {
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
- defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE">;
- defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE">;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE", f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE", i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_TFE", v2f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_TFE", v2i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE", v3f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE", v3i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZW_TFE", v4f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZW_TFE", v4i16>;
} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
foreach vt = Reg32Types.types in {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
index 2499182123c6b..15fa67c6ae717 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
@@ -257,5 +257,507 @@ define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
ret void
}
+define amdgpu_kernel void @load_v2f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v2f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v2f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v2f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v2f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v2f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v2f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<2 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<2 x half>, i32} %r, 0
+ %st = extractvalue {<2 x half>, i32} %r, 1
+ store <2 x half> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v2i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v2i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v2i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v2i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v2i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v2i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v2i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<2 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<2 x i16>, i32} %r, 0
+ %st = extractvalue {<2 x i16>, i32} %r, 1
+ store <2 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v4f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v4f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v4f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s1, s2, s3
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v4f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v4f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v4f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v4f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<4 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<4 x half>, i32} %r, 0
+ %st = extractvalue {<4 x half>, i32} %r, 1
+ store <4 x half> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v4i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v4i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v4i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s1, s2, s3
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v4i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v4i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v4i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v4i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<4 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<4 x i16>, i32} %r, 0
+ %st = extractvalue {<4 x i16>, i32} %r, 1
+ store <4 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
declare {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8), i32, i32, i32, i32)
declare {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<2 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<2 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<4 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<4 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i16i32s(ptr addrspace(8), i32, i32, i32, i32)
>From 3e6531622023b876ddb4986d36215f892c8639ec Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 7 Aug 2026 08:25:26 +0200
Subject: [PATCH 4/7] Address comment
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 54 ++++++++++-------------
1 file changed, 24 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 60e7825405074..4e455fcc93ae6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7769,6 +7769,26 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return SDValue();
}
+// TFE results are dword granular: value dwords followed by one status dword.
+static SDValue splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL,
+ SelectionDAG &DAG, SDValue &Status) {
+ LLVMContext &C = *DAG.getContext();
+ unsigned NumValueDWords = divideCeil(VT.getSizeInBits(), 32);
+ Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
+ DAG.getVectorIdxConstant(NumValueDWords, DL));
+ SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
+ SDValue ValueDWords =
+ NumValueDWords == 1
+ ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, ZeroIdx)
+ : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
+ EVT::getVectorVT(C, MVT::i32, NumValueDWords), Op,
+ ZeroIdx);
+ if (!VT.isVector() && VT.getSizeInBits() < 32)
+ ValueDWords =
+ DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), ValueDWords);
+ return DAG.getNode(ISD::BITCAST, DL, VT, ValueDWords);
+}
+
// Used for D16: Casts the result of an instruction into the right vector,
// packs values if loads return unpacked values.
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
@@ -7836,10 +7856,6 @@ SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
}
if (IsTFE) {
- // The hardware always returns TFE results at dword granularity: the data
- // dwords followed by one status dword. Load that combined vector, then
- // split it into the status and the D16 data before packing/truncating
- // the data the same way as the non-TFE case.
unsigned NumValueDWords = divideCeil(EquivLoadVT.getSizeInBits(), 32);
unsigned NumLoadDWords = NumValueDWords + 1;
EVT LoadVTList =
@@ -7847,22 +7863,8 @@ SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
SDVTList VTList = DAG.getVTList(LoadVTList, MVT::Other);
SDValue Load = DAG.getMemIntrinsicNode(
Opcode, DL, VTList, Ops, M->getMemoryVT(), M->getMemOperand());
- SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load,
- DAG.getVectorIdxConstant(NumValueDWords, DL));
- SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
- SDValue ValueDWords =
- NumValueDWords == 1
- ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load, ZeroIdx)
- : DAG.getNode(
- ISD::EXTRACT_SUBVECTOR, DL,
- EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumValueDWords),
- Load, ZeroIdx);
- // A scalar D16 result (f16/i16) occupies less than a full dword, so
- // truncate before bitcasting to the final scalar type.
- if (!EquivLoadVT.isVector() && EquivLoadVT.getSizeInBits() < 32)
- ValueDWords = DAG.getNode(ISD::TRUNCATE, DL,
- EquivLoadVT.changeTypeToInteger(), ValueDWords);
- SDValue Value = DAG.getNode(ISD::BITCAST, DL, EquivLoadVT, ValueDWords);
+ SDValue Status;
+ SDValue Value = splitTFEValueAndStatus(Load, EquivLoadVT, DL, DAG, Status);
SDValue Adjusted =
adjustLoadValueTypeImpl(Value, LoadVT, DL, DAG, Unpacked);
return DAG.getMergeValues({Adjusted, Status, SDValue(Load.getNode(), 1)},
@@ -12409,16 +12411,8 @@ SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
MF.getMachineMemOperand(MMO, 0, NumOpDWords * 4);
SDValue Op = getMemIntrinsicNode(Opcode, DL, OpDWordsVTList, Ops,
OpDWordsVT, OpDWordsMMO, DAG);
- SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
- DAG.getVectorIdxConstant(NumValueDWords, DL));
- SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
- SDValue ValueDWords =
- NumValueDWords == 1
- ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, ZeroIdx)
- : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
- EVT::getVectorVT(C, MVT::i32, NumValueDWords), Op,
- ZeroIdx);
- SDValue Value = DAG.getNode(ISD::BITCAST, DL, VT, ValueDWords);
+ SDValue Status;
+ SDValue Value = splitTFEValueAndStatus(Op, VT, DL, DAG, Status);
return DAG.getMergeValues({Value, Status, SDValue(Op.getNode(), 1)}, DL);
}
>From 83e0333e835527baced16b6a0fbfbde39b5e663f Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 11 Sep 2026 10:49:36 +0200
Subject: [PATCH 5/7] rename
---
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 2 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 3 +++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index ca627df0b44fe..0386fb51ec0e2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6838,7 +6838,7 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
- if (IsFormat && !IsTyped && IsD16 && IsTFE && ST.hasGFX90AInsts()) {
+ if (IsFormat && !IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "TFE D16 format buffer load is not supported on this GPU",
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index c42ca8e19ef9c..e0509055676c4 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -520,6 +520,9 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
// Has V_PK_MOV_B32 opcode
bool hasPkMovB32() const { return HasGFX90AInsts; }
+ // Has a hardware encoding for TFE D16 format buffer loads.
+ bool hasBufferTFEFormatD16() const { return !HasGFX90AInsts; }
+
bool hasFmaakFmamkF32Insts() const {
return getGeneration() >= GFX10 || hasGFX940Insts();
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4e455fcc93ae6..0d13336038065 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7903,7 +7903,7 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
- if (IsD16 && IsTFE && Subtarget->hasGFX90AInsts()) {
+ if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
"TFE D16 format buffer load is not supported on this GPU",
>From 36b2689b6f3aecfbdc489026f2827dd71d2d418f Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Fri, 11 Sep 2026 12:55:25 +0200
Subject: [PATCH 6/7] adj test
---
...dgcn.struct.ptr.buffer.load.format.d16.tfe.ll | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
index 15fa67c6ae717..5deaaf68856f6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
@@ -118,15 +118,15 @@ define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyz v[0:3], v0, s[0:3], 0 idxen tfe
; GFX8-UNPACKED-SDAG-NEXT: s_add_u32 s0, s4, 4
; GFX8-UNPACKED-SDAG-NEXT: s_addc_u32 s1, s5, 0
-; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s5
-; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-UNPACKED-SDAG-NEXT: s_mov_b32 s2, 0x1000504
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v9, s1
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s0
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s6
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s4
; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_perm_b32 v0, v0, v1, s2
; GFX8-UNPACKED-SDAG-NEXT: flat_store_short v[8:9], v2
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[6:7], v0
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[4:5], v3
@@ -266,13 +266,13 @@ define amdgpu_kernel void @load_v2f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: s_mov_b32 s0, 0x1000504
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
@@ -377,13 +377,13 @@ define amdgpu_kernel void @load_v2i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: s_mov_b32 s0, 0x1000504
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
>From eca61723982c704cf387b8420ce7287956167f1d Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Sun, 13 Sep 2026 20:27:11 +0200
Subject: [PATCH 7/7] simplify
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 119 ++++++++----------
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 14 +--
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 1 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 51 ++++----
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
...mdgcn.struct.ptr.buffer.format.i8.xfail.ll | 9 +-
6 files changed, 85 insertions(+), 113 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 88344f28999c3..397755a43854a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6845,6 +6845,22 @@ static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
.addMemOperand(MMO);
}
+static void buildTFEBufferLoad(unsigned Opc, ArrayRef<Register> ValueDsts,
+ Register StatusDst, Register RSrc,
+ Register VIndex, Register VOffset,
+ Register SOffset, unsigned ImmOffset,
+ unsigned Format, unsigned AuxiliaryData,
+ MachineMemOperand *MMO, bool IsTyped,
+ bool HasVIndex, MachineIRBuilder &B) {
+ LLT LoadTy = LLT::fixed_vector(ValueDsts.size() + 1, LLT::integer(32));
+ Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
+ buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
+ Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
+ SmallVector<Register, 5> Unmerge(ValueDsts);
+ Unmerge.push_back(StatusDst);
+ B.buildUnmerge(Unmerge, LoadDstReg);
+}
+
bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
LegalizerHelper &Helper,
bool IsFormat,
@@ -6931,7 +6947,7 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
- if (IsFormat && !IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
+ if (!IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "TFE D16 format buffer load is not supported on this GPU",
@@ -6978,85 +6994,60 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
}
if (IsTFE && IsD16 && Ty.isVector()) {
- // D16 dwords are packed (or, on unpacked-D16 targets, one element per
- // dword) at a different granularity than Ty's own bit width, so the
- // dwords loaded for the value do not necessarily cover Ty exactly (e.g.
- // v3i16 needs 2 dwords = 64 bits for its 48 bits of data). Load the
- // dwords, then repack/truncate down to Ty the same way the MUBUF/MIMG
- // TFE v3i16 case does.
+ // Value dwords need not cover Ty exactly: v3i16 needs 2 dwords for 48 bits.
const unsigned NumElts = Ty.getNumElements();
- const unsigned NumValueDWords =
- Unpacked ? NumElts : divideCeil(NumElts * 16, 32);
- const unsigned NumLoadDWords = NumValueDWords + 1;
- LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
- Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
- buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
- Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
+ const unsigned NumValueDWords = Unpacked ? NumElts : divideCeil(NumElts, 2);
- SmallVector<Register, 5> LoadElts;
+ SmallVector<Register, 4> ValueDWords;
for (unsigned I = 0; I != NumValueDWords; ++I)
- LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(I32));
- LoadElts.push_back(StatusDst);
- B.buildUnmerge(LoadElts, LoadDstReg);
- LoadElts.truncate(NumValueDWords);
+ ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
+ buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
+ SOffset, ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
+ HasVIndex, B);
- Register PackedI16;
if (Unpacked) {
- SmallVector<Register, 4> Repack;
- for (Register R : LoadElts)
- Repack.push_back(B.buildTrunc(LLT::integer(16), R).getReg(0));
- LLT RepackedTy = LLT::fixed_vector(NumValueDWords, LLT::integer(16));
- PackedI16 = B.buildMergeLikeInstr(RepackedTy, Repack).getReg(0);
+ for (Register &R : ValueDWords)
+ R = B.buildTrunc(EltTy, R).getReg(0);
+ B.buildMergeLikeInstr(Dst, ValueDWords);
} else {
- Register Merged;
- if (NumValueDWords == 1) {
- Merged = LoadElts[0];
+ Register Merged =
+ NumValueDWords == 1
+ ? ValueDWords[0]
+ : B.buildMergeLikeInstr(LLT::fixed_vector(NumValueDWords, I32),
+ ValueDWords)
+ .getReg(0);
+ LLT PackedTy = LLT::fixed_vector(NumValueDWords * 2, EltTy);
+ if (PackedTy == Ty) {
+ B.buildBitcast(Dst, Merged);
} else {
- LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
- Merged = B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ Register Packed = B.buildBitcast(PackedTy, Merged).getReg(0);
+ B.buildDeleteTrailingVectorElements(Dst, Packed);
}
- LLT PackedI16Ty = LLT::fixed_vector(NumValueDWords * 2, LLT::integer(16));
- PackedI16 = B.buildBitcast(PackedI16Ty, Merged).getReg(0);
}
-
- LLT PackedI16Ty = B.getMRI()->getType(PackedI16);
- LLT DstIntTy = Ty.changeElementType(LLT::integer(16));
- Register ValueI16 = PackedI16;
- if (PackedI16Ty.getNumElements() != NumElts) {
- ValueI16 = B.getMRI()->createGenericVirtualRegister(DstIntTy);
- B.buildDeleteTrailingVectorElements(ValueI16, PackedI16);
- }
- if (EltTy.isFloat())
- B.buildBitcast(Dst, ValueI16);
- else
- B.buildCopy(Dst, ValueI16);
} else if (IsTFE) {
- unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
- unsigned NumLoadDWords = NumValueDWords + 1;
- LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
- Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
- buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
- Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
- bool IsFloat = Ty.getScalarType().isFloat();
- LLT DstIntTy =
- IsFloat ? Ty.changeElementType(LLT::integer(EltTy.getSizeInBits()))
- : Ty;
+ const unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
Register DstInt =
- IsFloat ? B.getMRI()->createGenericVirtualRegister(DstIntTy) : Dst;
+ EltTy.isFloat() ? MRI.createGenericVirtualRegister(Ty.changeElementType(
+ LLT::integer(EltTy.getSizeInBits())))
+ : Dst;
if (MemTy.getSizeInBits() < 32) {
- Register ExtDst = B.getMRI()->createGenericVirtualRegister(I32);
- B.buildUnmerge({ExtDst, StatusDst}, LoadDstReg);
+ Register ExtDst = MRI.createGenericVirtualRegister(I32);
+ buildTFEBufferLoad(Opc, ExtDst, StatusDst, RSrc, VIndex, VOffset, SOffset,
+ ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
+ HasVIndex, B);
B.buildTrunc(DstInt, ExtDst);
} else if (NumValueDWords == 1) {
- B.buildUnmerge({DstInt, StatusDst}, LoadDstReg);
+ buildTFEBufferLoad(Opc, DstInt, StatusDst, RSrc, VIndex, VOffset, SOffset,
+ ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
+ HasVIndex, B);
} else {
- SmallVector<Register, 5> LoadElts;
+ SmallVector<Register, 4> ValueDWords;
for (unsigned I = 0; I != NumValueDWords; ++I)
- LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(I32));
- LoadElts.push_back(StatusDst);
- B.buildUnmerge(LoadElts, LoadDstReg);
- LoadElts.truncate(NumValueDWords);
- B.buildMergeLikeInstr(DstInt, LoadElts);
+ ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
+ buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
+ SOffset, ImmOffset, Format, AuxiliaryData, MMO,
+ IsTyped, HasVIndex, B);
+ B.buildMergeLikeInstr(DstInt, ValueDWords);
}
if (DstInt != Dst)
B.buildBitcast(Dst, DstInt);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 4e4f8acb10959..8d8cd71374211 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1284,7 +1284,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
- addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE},
+ addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE,
+ G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE},
StandardB)
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
@@ -1306,17 +1307,6 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
- addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE}, StandardB)
- .Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Any({{DivB160}, {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
- .Any({{UniB160},
- {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
-
addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
// waterfall expansion is part of S_BUF_to_BUF
.Any({{UniB32}, {{SgprB32}, {SgprV4S32, Sgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 250cc7efdcfdb..34ae87232a81b 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -528,7 +528,6 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
// Has V_PK_MOV_B32 opcode
bool hasPkMovB32() const { return HasGFX90AInsts; }
- // Has a hardware encoding for TFE D16 format buffer loads.
bool hasBufferTFEFormatD16() const { return !HasGFX90AInsts; }
bool hasFmaakFmamkF32Insts() const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8930789ce6a19..2b88ac654ff39 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7838,12 +7838,12 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
// TFE results are dword granular: value dwords followed by one status dword.
-static SDValue splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL,
- SelectionDAG &DAG, SDValue &Status) {
+static std::pair<SDValue, SDValue>
+splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG) {
LLVMContext &C = *DAG.getContext();
unsigned NumValueDWords = divideCeil(VT.getSizeInBits(), 32);
- Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
- DAG.getVectorIdxConstant(NumValueDWords, DL));
+ SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op,
+ DAG.getVectorIdxConstant(NumValueDWords, DL));
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
SDValue ValueDWords =
NumValueDWords == 1
@@ -7854,7 +7854,7 @@ static SDValue splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL,
if (!VT.isVector() && VT.getSizeInBits() < 32)
ValueDWords =
DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), ValueDWords);
- return DAG.getNode(ISD::BITCAST, DL, VT, ValueDWords);
+ return {DAG.getNode(ISD::BITCAST, DL, VT, ValueDWords), Status};
}
// Used for D16: Casts the result of an instruction into the right vector,
@@ -7903,10 +7903,10 @@ static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
SelectionDAG &DAG,
ArrayRef<SDValue> Ops,
- bool IsIntrinsic,
- bool IsTFE) const {
+ bool IsIntrinsic) const {
SDLoc DL(M);
+ bool IsTFE = M->getNumValues() == 3;
bool Unpacked = Subtarget->hasUnpackedD16VMem();
EVT LoadVT = M->getValueType(0);
@@ -7925,18 +7925,15 @@ SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
if (IsTFE) {
unsigned NumValueDWords = divideCeil(EquivLoadVT.getSizeInBits(), 32);
- unsigned NumLoadDWords = NumValueDWords + 1;
- EVT LoadVTList =
- EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumLoadDWords);
- SDVTList VTList = DAG.getVTList(LoadVTList, MVT::Other);
+ EVT LoadDWordsVT =
+ EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumValueDWords + 1);
+ SDVTList VTList = DAG.getVTList(LoadDWordsVT, MVT::Other);
SDValue Load = DAG.getMemIntrinsicNode(
Opcode, DL, VTList, Ops, M->getMemoryVT(), M->getMemOperand());
- SDValue Status;
- SDValue Value = splitTFEValueAndStatus(Load, EquivLoadVT, DL, DAG, Status);
+ auto [Value, Status] = splitTFEValueAndStatus(Load, EquivLoadVT, DL, DAG);
SDValue Adjusted =
adjustLoadValueTypeImpl(Value, LoadVT, DL, DAG, Unpacked);
- return DAG.getMergeValues({Adjusted, Status, SDValue(Load.getNode(), 1)},
- DL);
+ return DAG.getMergeValues({Adjusted, Status, Load.getValue(1)}, DL);
}
// Change from v4f16/v2f16 to EquivLoadVT.
@@ -7976,20 +7973,19 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
DAG.getMachineFunction().getFunction(),
"TFE D16 format buffer load is not supported on this GPU",
DL.getDebugLoc()));
- return DAG.getMergeValues(
- {DAG.getPOISON(LoadVT), DAG.getPOISON(MVT::i32), M->getOperand(0)}, DL);
+ return DAG.getErrorMergeValues({M->value_begin(), M->value_end()},
+ M->getOperand(0), DL);
}
- unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
- : AMDGPUISD::BUFFER_LOAD_FORMAT)
- : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
- : AMDGPUISD::BUFFER_LOAD;
+ unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
+ : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
+ : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
+ : AMDGPUISD::BUFFER_LOAD_FORMAT)
+ : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
+ : AMDGPUISD::BUFFER_LOAD;
- if (IsD16) {
- return adjustLoadValueType(IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
- : AMDGPUISD::BUFFER_LOAD_FORMAT_D16,
- M, DAG, Ops, /*IsIntrinsic=*/false, IsTFE);
- }
+ if (IsD16)
+ return adjustLoadValueType(Opc, M, DAG, Ops);
// Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
@@ -12497,8 +12493,7 @@ SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
MF.getMachineMemOperand(MMO, 0, NumOpDWords * 4);
SDValue Op = getMemIntrinsicNode(Opcode, DL, OpDWordsVTList, Ops,
OpDWordsVT, OpDWordsMMO, DAG);
- SDValue Status;
- SDValue Value = splitTFEValueAndStatus(Op, VT, DL, DAG, Status);
+ auto [Value, Status] = splitTFEValueAndStatus(Op, VT, DL, DAG);
return DAG.getMergeValues({Value, Status, SDValue(Op.getNode(), 1)}, DL);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index a1e1722f65c0b..421954ebf3363 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -141,8 +141,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M, SelectionDAG &DAG,
- ArrayRef<SDValue> Ops, bool IsIntrinsic = false,
- bool IsTFE = false) const;
+ ArrayRef<SDValue> Ops,
+ bool IsIntrinsic = false) const;
SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
ArrayRef<SDValue> Ops) const;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
index 9f5efef3796f2..536e33313b6e8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
@@ -1,7 +1,7 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
-; RUN: not llc -global-isel=0 -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
-; RUN: not llc -global-isel -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
+; RUN: not llc -global-isel=0 -mtriple=amdgpu9.0a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
+; RUN: not llc -global-isel -mtriple=amdgpu9.0a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
; An i8 buffer.load.format / buffer.store.format has no corresponding real
; instruction (no byte-granularity format access exists in hardware), so both
@@ -23,10 +23,7 @@ define amdgpu_ps void @store_i8(ptr addrspace(8) inreg %rsrc, i8 %data, i32 %ind
ret void
}
-; D16 buffer.load.format combined with TFE has no real hardware encoding on
-; gfx90a, so it must be refused there. Other targets (gfx8/gfx10/gfx11/gfx12)
-; have real TFE encodings and are covered by
-; llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll instead.
+; D16 buffer.load.format with TFE has no hardware encoding on gfx90a.
; CHECK-NOT: error: {{.*}}TFE D16 format buffer load
; GFX90A: error: {{.*}}TFE D16 format buffer load is not supported on this GPU
define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
More information about the llvm-commits
mailing list