[llvm] [AMDGPU] Support TFE D16 format buffer loads on gfx8/10/11/12 (PR #211465)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 20:46:48 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/211465
>From 2b514271414d3a0f5642705eb1cca1a633ba6b72 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 23 Jul 2026 08:00:25 +0200
Subject: [PATCH 1/3] [AMDGPU] Reject unsupported TFE D16 format buffer loads
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 10 ++++++++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 23 ++++++++++++++-----
...mdgcn.struct.ptr.buffer.format.i8.xfail.ll | 23 +++++++++++++++++++
3 files changed, 50 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 87e85bef7c701..8db7fd4d5541d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6901,6 +6901,16 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
+ if (IsFormat && !IsTyped && IsD16 && IsTFE) {
+ const Function &Fn = B.getMF().getFunction();
+ Fn.getContext().diagnose(DiagnosticInfoUnsupported(
+ Fn, "unsupported TFE D16 format buffer load", MI.getDebugLoc()));
+ B.buildUndef(Dst);
+ B.buildUndef(StatusDst);
+ MI.eraseFromParent();
+ return true;
+ }
+
std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
unsigned Opc;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8156de7a24b01..b2ce3337bac02 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7835,6 +7835,14 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
+ if (IsD16 && IsTFE) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported TFE D16 format buffer load", DL.getDebugLoc()));
+ return DAG.getMergeValues(
+ {DAG.getPOISON(LoadVT), DAG.getPOISON(MVT::i32), M->getOperand(0)}, DL);
+ }
+
unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
: AMDGPUISD::BUFFER_LOAD_FORMAT)
: IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
@@ -7855,12 +7863,15 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
}
EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
- SDVTList VTList = DAG.getVTList(CastVT, MVT::Other);
+ SDVTList VTList = IsTFE ? DAG.getVTList(CastVT, MVT::i32, MVT::Other)
+ : DAG.getVTList(CastVT, MVT::Other);
SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
M->getMemOperand(), DAG);
- return DAG.getMergeValues(
- {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)},
- DL);
+ SDValue Data = DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode);
+ if (IsTFE)
+ return DAG.getMergeValues({Data, MemNode.getValue(1), MemNode.getValue(2)},
+ DL);
+ return DAG.getMergeValues({Data, MemNode.getValue(1)}, DL);
}
static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
@@ -8407,8 +8418,8 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(Res.getOperand(I));
}
} else {
- Results.push_back(Res);
- Results.push_back(Res.getValue(1));
+ for (unsigned I = 0; I < N->getNumValues(); ++I)
+ Results.push_back(Res.getValue(I));
}
return;
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
index f5f8471f4dc1f..df137c5df98ea 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
@@ -18,3 +18,26 @@ define amdgpu_ps void @store_i8(ptr addrspace(8) inreg %rsrc, i8 %data, i32 %ind
call void @llvm.amdgcn.struct.ptr.buffer.store.format.i8(i8 %data, ptr addrspace(8) %rsrc, i32 %index, i32 0, i32 0, i32 0)
ret void
}
+
+; A D16 buffer.load.format combined with TFE has no corresponding real
+; instruction (no TFE variant of the D16 format load opcodes exists in
+; hardware), so it must be refused instead of miscounting SDNode results.
+; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+ %r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<3 x i16>, i32} %r, 0
+ %st = extractvalue {<3 x i16>, i32} %r, 1
+ store <3 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+ %r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {half, i32} %r, 0
+ %st = extractvalue {half, i32} %r, 1
+ store half %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
>From e8d9ae05ac893900d832624e41173e8fbbdc2800 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 30 Jul 2026 11:40:46 +0200
Subject: [PATCH 2/3] Address comment
---
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 1 +
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 63 ++++-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 8 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 2 +
llvm/lib/Target/AMDGPU/BUFInstructions.td | 6 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 46 ++-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 3 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 1 +
...mdgcn.struct.ptr.buffer.format.i8.xfail.ll | 16 +-
...n.struct.ptr.buffer.load.format.d16.tfe.ll | 261 ++++++++++++++++++
11 files changed, 395 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 5df9834f4ef80..df4a197009bf6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -296,6 +296,7 @@ def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_SBYTE_TFE, SIbuffer_load_byte_tfe>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT, SIbuffer_load_format>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_TFE, SIbuffer_load_format_tfe>;
def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16, SIbuffer_load_format_d16>;
+def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE, SIbuffer_load_format_d16_tfe>;
def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT, SItbuffer_load>;
def : GINodeEquiv<G_AMDGPU_TBUFFER_LOAD_FORMAT_D16, SItbuffer_load_d16>;
def : GINodeEquiv<G_AMDGPU_BUFFER_STORE, SIbuffer_store>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c5cea131dc64b..3205213c86f04 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6862,10 +6862,11 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
return true;
}
- if (IsFormat && !IsTyped && IsD16 && IsTFE) {
+ if (IsFormat && !IsTyped && IsD16 && IsTFE && ST.hasGFX90AInsts()) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
- Fn, "unsupported TFE D16 format buffer load", MI.getDebugLoc()));
+ Fn, "TFE D16 format buffer load is not supported on this GPU",
+ MI.getDebugLoc()));
B.buildUndef(Dst);
B.buildUndef(StatusDst);
MI.eraseFromParent();
@@ -6884,9 +6885,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
} else if (IsFormat) {
if (IsD16) {
- if (IsTFE)
- return false;
- Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
+ Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE
+ : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
} else {
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
@@ -6908,7 +6908,58 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
}
}
- if (IsTFE) {
+ if (IsTFE && IsD16 && Ty.isVector()) {
+ // D16 dwords are packed (or, on unpacked-D16 targets, one element per
+ // dword) at a different granularity than Ty's own bit width, so the
+ // dwords loaded for the value do not necessarily cover Ty exactly (e.g.
+ // v3i16 needs 2 dwords = 64 bits for its 48 bits of data). Load the
+ // dwords, then repack/truncate down to Ty the same way the MUBUF/MIMG
+ // TFE v3i16 case does.
+ const unsigned NumElts = Ty.getNumElements();
+ const unsigned NumValueDWords =
+ Unpacked ? NumElts : divideCeil(NumElts * 16, 32);
+ const unsigned NumLoadDWords = NumValueDWords + 1;
+ LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
+ Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
+ buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
+ Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
+
+ SmallVector<Register, 5> LoadElts;
+ for (unsigned I = 0; I != NumValueDWords; ++I)
+ LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(I32));
+ LoadElts.push_back(StatusDst);
+ B.buildUnmerge(LoadElts, LoadDstReg);
+ LoadElts.truncate(NumValueDWords);
+
+ Register PackedI16;
+ if (Unpacked) {
+ SmallVector<Register, 4> Repack;
+ for (Register R : LoadElts)
+ Repack.push_back(B.buildTrunc(LLT::integer(16), R).getReg(0));
+ LLT RepackedTy = LLT::fixed_vector(NumValueDWords, LLT::integer(16));
+ PackedI16 = B.buildMergeLikeInstr(RepackedTy, Repack).getReg(0);
+ } else {
+ LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
+ Register Merged =
+ NumValueDWords == 1
+ ? LoadElts[0]
+ : B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ LLT PackedI16Ty = LLT::fixed_vector(NumValueDWords * 2, LLT::integer(16));
+ PackedI16 = B.buildBitcast(PackedI16Ty, Merged).getReg(0);
+ }
+
+ LLT PackedI16Ty = B.getMRI()->getType(PackedI16);
+ LLT DstIntTy = Ty.changeElementType(LLT::integer(16));
+ Register ValueI16 = PackedI16;
+ if (PackedI16Ty.getNumElements() != NumElts) {
+ ValueI16 = B.getMRI()->createGenericVirtualRegister(DstIntTy);
+ B.buildDeleteTrailingVectorElements(ValueI16, PackedI16);
+ }
+ if (EltTy.isFloat())
+ B.buildBitcast(Dst, ValueI16);
+ else
+ B.buildCopy(Dst, ValueI16);
+ } else if (IsTFE) {
unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
unsigned NumLoadDWords = NumValueDWords + 1;
LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index b9bc59a79a642..827e491b97abc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1296,6 +1296,14 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+ addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE}, StandardB)
+ .Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+
addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
// waterfall expansion is part of S_BUF_to_BUF
.Any({{UniB32}, {{SgprB32}, {SgprV4S32, Sgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 21472bb886196..a4edd4b699f27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3100,6 +3100,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
+ case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
case AMDGPU::G_AMDGPU_BUFFER_STORE:
@@ -4499,6 +4500,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
+ case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 5faa478407c34..f20648df617e4 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1468,6 +1468,9 @@ let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
+
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80_TFE">;
} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
@@ -1486,6 +1489,9 @@ let True16Predicate = UseRealTrue16Insts in {
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
+
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE">;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE">;
} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
foreach vt = Reg32Types.types in {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6f52f7c098b97..c8bfc808cc2f3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7792,7 +7792,8 @@ static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
SelectionDAG &DAG,
ArrayRef<SDValue> Ops,
- bool IsIntrinsic) const {
+ bool IsIntrinsic,
+ bool IsTFE) const {
SDLoc DL(M);
bool Unpacked = Subtarget->hasUnpackedD16VMem();
@@ -7811,6 +7812,40 @@ SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
}
}
+ if (IsTFE) {
+ // The hardware always returns TFE results at dword granularity: the data
+ // dwords followed by one status dword. Load that combined vector, then
+ // split it into the status and the D16 data before packing/truncating
+ // the data the same way as the non-TFE case.
+ unsigned NumValueDWords = divideCeil(EquivLoadVT.getSizeInBits(), 32);
+ unsigned NumLoadDWords = NumValueDWords + 1;
+ EVT LoadVTList =
+ EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumLoadDWords);
+ SDVTList VTList = DAG.getVTList(LoadVTList, MVT::Other);
+ SDValue Load = DAG.getMemIntrinsicNode(
+ Opcode, DL, VTList, Ops, M->getMemoryVT(), M->getMemOperand());
+ SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load,
+ DAG.getVectorIdxConstant(NumValueDWords, DL));
+ SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
+ SDValue ValueDWords =
+ NumValueDWords == 1
+ ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Load, ZeroIdx)
+ : DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL,
+ EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumValueDWords),
+ Load, ZeroIdx);
+ // A scalar D16 result (f16/i16) occupies less than a full dword, so
+ // truncate before bitcasting to the final scalar type.
+ if (!EquivLoadVT.isVector() && EquivLoadVT.getSizeInBits() < 32)
+ ValueDWords = DAG.getNode(ISD::TRUNCATE, DL,
+ EquivLoadVT.changeTypeToInteger(), ValueDWords);
+ SDValue Value = DAG.getNode(ISD::BITCAST, DL, EquivLoadVT, ValueDWords);
+ SDValue Adjusted =
+ adjustLoadValueTypeImpl(Value, LoadVT, DL, DAG, Unpacked);
+ return DAG.getMergeValues({Adjusted, Status, SDValue(Load.getNode(), 1)},
+ DL);
+ }
+
// Change from v4f16/v2f16 to EquivLoadVT.
SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
@@ -7843,10 +7878,11 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
- if (IsD16 && IsTFE) {
+ if (IsD16 && IsTFE && Subtarget->hasGFX90AInsts()) {
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
- "unsupported TFE D16 format buffer load", DL.getDebugLoc()));
+ "TFE D16 format buffer load is not supported on this GPU",
+ DL.getDebugLoc()));
return DAG.getMergeValues(
{DAG.getPOISON(LoadVT), DAG.getPOISON(MVT::i32), M->getOperand(0)}, DL);
}
@@ -7857,7 +7893,9 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
: AMDGPUISD::BUFFER_LOAD;
if (IsD16) {
- return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
+ return adjustLoadValueType(IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
+ : AMDGPUISD::BUFFER_LOAD_FORMAT_D16,
+ M, DAG, Ops, /*IsIntrinsic=*/false, IsTFE);
}
// Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 86653fe6920c5..e56aaf37270d2 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -133,9 +133,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
- SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
- SelectionDAG &DAG, ArrayRef<SDValue> Ops,
- bool IsIntrinsic = false) const;
+ SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M, SelectionDAG &DAG,
+ ArrayRef<SDValue> Ops, bool IsIntrinsic = false,
+ bool IsTFE = false) const;
SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
ArrayRef<SDValue> Ops) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 8208a6ea2d5b5..066bfe564969c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -172,6 +172,9 @@ def SIbuffer_load_format_tfe : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_TFE", SDTB
def SIbuffer_load_format_d16 : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_D16",
SDTBufferLoad,
[SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
+def SIbuffer_load_format_d16_tfe : SDNode <"AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE",
+ SDTBufferLoad,
+ [SDNPMemOperand, SDNPHasChain, SDNPMayLoad]>;
def SDTBufferStore : SDTypeProfile<0, 8,
[ // vdata
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f04373d3163a9..9ee8af5cd3b98 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -4775,6 +4775,7 @@ def G_AMDGPU_BUFFER_LOAD_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction;
+def G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE : BufferLoadGenericInstruction;
def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction;
def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
index df137c5df98ea..9f5efef3796f2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.format.i8.xfail.ll
@@ -1,11 +1,14 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
; RUN: not llc -global-isel -mtriple=amdgpu9.00 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=0 -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
+; RUN: not llc -global-isel -mtriple=amdgpu9.0a -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX90A %s
; An i8 buffer.load.format / buffer.store.format has no corresponding real
; instruction (no byte-granularity format access exists in hardware), so both
; SelectionDAG and GlobalISel must refuse to lower it.
; CHECK: error: {{.*}}unsupported sub-dword format buffer load
+; GFX90A: error: {{.*}}unsupported sub-dword format buffer load
define amdgpu_ps float @load_i8(ptr addrspace(8) inreg %rsrc) {
%data = call i8 @llvm.amdgcn.struct.ptr.buffer.load.format.i8(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%zext = zext i8 %data to i32
@@ -14,15 +17,18 @@ define amdgpu_ps float @load_i8(ptr addrspace(8) inreg %rsrc) {
}
; CHECK: error: {{.*}}unsupported sub-dword format buffer store
+; GFX90A: error: {{.*}}unsupported sub-dword format buffer store
define amdgpu_ps void @store_i8(ptr addrspace(8) inreg %rsrc, i8 %data, i32 %index) {
call void @llvm.amdgcn.struct.ptr.buffer.store.format.i8(i8 %data, ptr addrspace(8) %rsrc, i32 %index, i32 0, i32 0, i32 0)
ret void
}
-; A D16 buffer.load.format combined with TFE has no corresponding real
-; instruction (no TFE variant of the D16 format load opcodes exists in
-; hardware), so it must be refused instead of miscounting SDNode results.
-; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+; D16 buffer.load.format combined with TFE has no real hardware encoding on
+; gfx90a, so it must be refused there. Other targets (gfx8/gfx10/gfx11/gfx12)
+; have real TFE encodings and are covered by
+; llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll instead.
+; CHECK-NOT: error: {{.*}}TFE D16 format buffer load
+; GFX90A: error: {{.*}}TFE D16 format buffer load is not supported on this GPU
define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
%r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue {<3 x i16>, i32} %r, 0
@@ -32,7 +38,7 @@ define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
ret void
}
-; CHECK: error: {{.*}}unsupported TFE D16 format buffer load
+; GFX90A: error: {{.*}}TFE D16 format buffer load is not supported on this GPU
define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
%r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue {half, i32} %r, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
new file mode 100644
index 0000000000000..2499182123c6b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-UNPACKED-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-UNPACKED-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu8.10 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-PACKED-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu8.10 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-PACKED-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX9-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX9-GISEL %s
+
+define amdgpu_kernel void @load_f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_x v[0:1], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_short v[2:3], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_x v[0:1], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_x v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_short v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_x v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_x v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_x v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {half, i32} %r, 0
+ %st = extractvalue {half, i32} %r, 1
+ store half %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v3i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyz v[0:3], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: s_add_u32 s0, s4, 4
+; GFX8-UNPACKED-SDAG-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v9, s1
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_short v[8:9], v2
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[6:7], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[4:5], v3
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v3i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyz v[0:3], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: s_add_u32 s0, s4, 2
+; GFX8-UNPACKED-GISEL-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v9, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s0
+; GFX8-UNPACKED-GISEL-NEXT: s_add_u32 s0, s4, 4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-UNPACKED-GISEL-NEXT: s_addc_u32 s1, s5, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v11, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v10, s0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[4:5], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[8:9], v1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_short v[10:11], v2
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[6:7], v3
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v3i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyz v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: s_add_u32 s0, s12, 4
+; GFX8-PACKED-SDAG-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v8, s1
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v7, s0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s15
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s13
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_short v[7:8], v1
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[3:4], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v3i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyz v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-PACKED-GISEL-NEXT: s_add_u32 s0, s12, 2
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-PACKED-GISEL-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-PACKED-GISEL-NEXT: s_add_u32 s0, s12, 4
+; GFX8-PACKED-GISEL-NEXT: s_addc_u32 s1, s13, 0
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-PACKED-GISEL-NEXT: flat_store_short v[3:4], v1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v3i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyz v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_short v0, v2, s[12:13] offset:4
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v3i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyz v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_short v0, v2, s[12:13] offset:4
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[12:13] offset:2
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<3 x i16>, i32} %r, 0
+ %st = extractvalue {<3 x i16>, i32} %r, 1
+ store <3 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+declare {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8), i32, i32, i32, i32)
>From be54b711b15a042a8b9ea08c1b551b010a307027 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 4 Aug 2026 17:42:12 +0200
Subject: [PATCH 3/3] Address comments
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 12 +-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 5 +-
llvm/lib/Target/AMDGPU/BUFInstructions.td | 12 +-
...n.struct.ptr.buffer.load.format.d16.tfe.ll | 502 ++++++++++++++++++
4 files changed, 523 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 3205213c86f04..d05e9f0dfd98b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6939,11 +6939,13 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
LLT RepackedTy = LLT::fixed_vector(NumValueDWords, LLT::integer(16));
PackedI16 = B.buildMergeLikeInstr(RepackedTy, Repack).getReg(0);
} else {
- LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
- Register Merged =
- NumValueDWords == 1
- ? LoadElts[0]
- : B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ Register Merged;
+ if (NumValueDWords == 1) {
+ Merged = LoadElts[0];
+ } else {
+ LLT MergedTy = LLT::fixed_vector(NumValueDWords, I32);
+ Merged = B.buildMergeLikeInstr(MergedTy, LoadElts).getReg(0);
+ }
LLT PackedI16Ty = LLT::fixed_vector(NumValueDWords * 2, LLT::integer(16));
PackedI16 = B.buildBitcast(PackedI16Ty, Merged).getReg(0);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 827e491b97abc..2cc0a55cb3368 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1302,7 +1302,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
- .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
+ .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
+ .Any({{DivB160}, {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
+ .Any({{UniB160},
+ {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
addRulesForGOpcs({G_AMDGPU_S_BUFFER_LOAD})
// waterfall expansion is part of S_BUF_to_BUF
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index f20648df617e4..ffc31aedc480e 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1470,7 +1470,9 @@ let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80_TFE">;
defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80_TFE">;
+ defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16_tfe, v5i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_TFE">;
} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
@@ -1490,8 +1492,14 @@ let True16Predicate = UseRealTrue16Insts in {
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
- defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE">;
- defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE">;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE", f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_X_TFE", i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_TFE", v2f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_TFE", v2i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE", v3f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_TFE", v3i16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZW_TFE", v4f16>;
+ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16_tfe, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZW_TFE", v4i16>;
} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
foreach vt = Reg32Types.types in {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
index 2499182123c6b..15fa67c6ae717 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.d16.tfe.ll
@@ -257,5 +257,507 @@ define amdgpu_kernel void @load_v3i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addr
ret void
}
+define amdgpu_kernel void @load_v2f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v2f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v2f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v2f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v2f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v2f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v2f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<2 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<2 x half>, i32} %r, 0
+ %st = extractvalue {<2 x half>, i32} %r, 1
+ store <2 x half> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v2i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v2i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v2i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:2], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[3:4], v0
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v2i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v2i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xy v[0:1], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[2:3], v0
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[4:5], v1
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v2i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v2i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xy v[1:2], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v2, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<2 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<2 x i16>, i32} %r, 0
+ %st = extractvalue {<2 x i16>, i32} %r, 1
+ store <2 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v4f16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v4f16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v4f16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s1, s2, s3
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v4f16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v4f16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v4f16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v4f16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<4 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<4 x half>, i32} %r, 0
+ %st = extractvalue {<4 x half>, i32} %r, 1
+ store <4 x half> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
+define amdgpu_kernel void @load_v4i16_tfe(ptr addrspace(8) inreg %rsrc, ptr addrspace(1) %out, ptr addrspace(1) %status) {
+; GFX8-UNPACKED-SDAG-LABEL: load_v4i16_tfe:
+; GFX8-UNPACKED-SDAG: ; %bb.0:
+; GFX8-UNPACKED-SDAG-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-SDAG-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-SDAG-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-UNPACKED-GISEL-LABEL: load_v4i16_tfe:
+; GFX8-UNPACKED-GISEL: ; %bb.0:
+; GFX8-UNPACKED-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:4], v0, s[0:3], 0 idxen tfe
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v6, s5
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v8, s7
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-UNPACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-UNPACKED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-UNPACKED-GISEL-NEXT: s_or_b32 s1, s2, s3
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-UNPACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dwordx2 v[5:6], v[0:1]
+; GFX8-UNPACKED-GISEL-NEXT: flat_store_dword v[7:8], v4
+; GFX8-UNPACKED-GISEL-NEXT: s_endpgm
+;
+; GFX8-PACKED-SDAG-LABEL: load_v4i16_tfe:
+; GFX8-PACKED-SDAG: ; %bb.0:
+; GFX8-PACKED-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-SDAG-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-SDAG-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-SDAG-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-SDAG-NEXT: s_endpgm
+;
+; GFX8-PACKED-GISEL-LABEL: load_v4i16_tfe:
+; GFX8-PACKED-GISEL: ; %bb.0:
+; GFX8-PACKED-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: buffer_load_format_d16_xyzw v[0:2], v0, s[8:11], 0 idxen tfe
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s12
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v4, s13
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v5, s14
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v6, s15
+; GFX8-PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-PACKED-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-PACKED-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-PACKED-GISEL-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-PACKED-GISEL-NEXT: flat_store_dword v[5:6], v2
+; GFX8-PACKED-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: load_v4i16_tfe:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-SDAG-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: load_v4i16_tfe:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_load_format_d16_xyzw v[1:3], v0, s[8:11], 0 idxen tfe
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT: global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX9-GISEL-NEXT: global_store_dword v0, v3, s[14:15]
+; GFX9-GISEL-NEXT: s_endpgm
+ %r = call {<4 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i16i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
+ %data = extractvalue {<4 x i16>, i32} %r, 0
+ %st = extractvalue {<4 x i16>, i32} %r, 1
+ store <4 x i16> %data, ptr addrspace(1) %out
+ store i32 %st, ptr addrspace(1) %status
+ ret void
+}
+
declare {half, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f16i32s(ptr addrspace(8), i32, i32, i32, i32)
declare {<3 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<2 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<2 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<4 x half>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f16i32s(ptr addrspace(8), i32, i32, i32, i32)
+declare {<4 x i16>, i32} @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i16i32s(ptr addrspace(8), i32, i32, i32, i32)
More information about the llvm-commits
mailing list