[llvm] [AMDGPU] Support i16 element types for tbuffer D16 load/store (PR #201420)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 06:12:15 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/201420
>From 4c92be41826d11609b3af4cf90bda01afd32ddc9 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 19:30:51 +0200
Subject: [PATCH 1/3] [AMDGPU] Support i16 element types for tbuffer D16
load/store
Detect D16 by element bit width rather than matching f16 specifically, so integer i16 elements take the same packed/unpacked D16 path
---
llvm/lib/Target/AMDGPU/BUFInstructions.td | 10 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll | 146 ++++++++++++++++++
.../llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll | 33 ++++
4 files changed, 193 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 909679b899714..d4f1809ab87d8 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2354,6 +2354,7 @@ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">
let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
+ defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
@@ -2362,10 +2363,14 @@ let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">;
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i16, "TBUFFER_LOAD_FORMAT_D16_X">;
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">;
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i16, "TBUFFER_LOAD_FORMAT_D16_XY">;
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
} // End HasPackedD16VMem,HasMTBUFInsts.
multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
@@ -2428,6 +2433,7 @@ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZ
let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
+ defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
@@ -2436,10 +2442,14 @@ let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">;
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i16, "TBUFFER_STORE_FORMAT_D16_X">;
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">;
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i16, "TBUFFER_STORE_FORMAT_D16_XY">;
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3i16>;
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
} // End HasPackedD16VMem,HasMTBUFInsts.
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b393bb904e75b..d46c9d63f3404 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -11453,7 +11453,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarType().getSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11480,7 +11480,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(1, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarType().getSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -12134,7 +12134,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarType().getSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12162,7 +12162,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarType().getSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
index a9ea440779a49..ad06215c733a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
@@ -137,7 +137,153 @@ main_body:
ret half %elt
}
+; Integer (i16) element types must use the same D16 path as f16.
+define amdgpu_ps i16 @tbuffer_load_i16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_i16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x00,0xb4,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_readfirstlane_b32 s0, v0 ; encoding: [0x00,0x05,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_i16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_i16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_i16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call i16 @llvm.amdgcn.raw.ptr.tbuffer.load.i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ ret i16 %data
+}
+
+define amdgpu_ps i16 @tbuffer_load_v2i16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_v2i16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_xy v[0:1], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0xb4,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_readfirstlane_b32 s0, v1 ; encoding: [0x01,0x05,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_v2i16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; PREGFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_v2i16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_v2i16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call <2 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v2i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ %elt = extractelement <2 x i16> %data, i32 1
+ ret i16 %elt
+}
+
+define amdgpu_ps i16 @tbuffer_load_v3i16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_v3i16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_xyz v[0:2], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x00,0xb5,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_readfirstlane_b32 s0, v2 ; encoding: [0x02,0x05,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_v3i16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_xyz v[0:1], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v1
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_v3i16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_v3i16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call <3 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v3i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ %elt = extractelement <3 x i16> %data, i32 2
+ ret i16 %elt
+}
+
+define amdgpu_ps i16 @tbuffer_load_v4i16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_v4i16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0xb5,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_readfirstlane_b32 s0, v3 ; encoding: [0x03,0x05,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_v4i16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; PREGFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_v4i16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX10-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_v4i16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-PACKED-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call <4 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v4i16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ %elt = extractelement <4 x i16> %data, i32 3
+ ret i16 %elt
+}
+
declare half @llvm.amdgcn.raw.ptr.tbuffer.load.f16(ptr addrspace(8), i32, i32, i32, i32)
declare <2 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v2f16(ptr addrspace(8), i32, i32, i32, i32)
declare <3 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v3f16(ptr addrspace(8), i32, i32, i32, i32)
declare <4 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v4f16(ptr addrspace(8), i32, i32, i32, i32)
+declare i16 @llvm.amdgcn.raw.ptr.tbuffer.load.i16(ptr addrspace(8), i32, i32, i32, i32)
+declare <2 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v2i16(ptr addrspace(8), i32, i32, i32, i32)
+declare <3 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v3i16(ptr addrspace(8), i32, i32, i32, i32)
+declare <4 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v4i16(ptr addrspace(8), i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
index 9a51b1206366f..752b673ca6758 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
@@ -203,10 +203,43 @@ main_body:
ret void
}
+; Integer (i16) element types must use the same D16 path as f16.
+define amdgpu_ps void @tbuffer_store_v3i16(ptr addrspace(8) inreg %rsrc, <3 x i16> %data) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_store_v3i16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; PREGFX10-UNPACKED-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; PREGFX10-UNPACKED-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; PREGFX10-UNPACKED-NEXT: tbuffer_store_format_d16_xyz v[0:2], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-UNPACKED-NEXT: s_endpgm
+;
+; PREGFX10-PACKED-LABEL: tbuffer_store_v3i16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; PREGFX10-PACKED-NEXT: tbuffer_store_format_d16_xyz v[0:1], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-PACKED-NEXT: s_endpgm
+;
+; GFX10-PACKED-LABEL: tbuffer_store_v3i16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-PACKED-NEXT: tbuffer_store_format_d16_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED]
+; GFX10-PACKED-NEXT: s_endpgm
+;
+; GFX11-PACKED-LABEL: tbuffer_store_v3i16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM]
+; GFX11-PACKED-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.raw.ptr.tbuffer.store.v3i16(<3 x i16> %data, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0)
+ ret void
+}
+
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v3f16(<3 x half>, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f16(<4 x half>, ptr addrspace(8), i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v3i16(<3 x i16>, ptr addrspace(8), i32, i32, i32, i32)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11-PACKED-FAKE16: {{.*}}
; GFX11-PACKED-TRUE16: {{.*}}
>From 5c3c4787daf8af86545b41f608d63de61608d374 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <gooddoog at student.su>
Date: Thu, 4 Jun 2026 18:50:43 +0200
Subject: [PATCH 2/3] Address review comments
---
llvm/lib/Target/AMDGPU/BUFInstructions.td | 92 +++++++++-------
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll | 101 ++++++++++++++++++
.../llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll | 85 +++++++++++++++
4 files changed, 246 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index d4f1809ab87d8..6753989636055 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2342,19 +2342,24 @@ multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
}
let OtherPredicates = [HasMTBUFInsts] in {
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
-defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
+foreach vt = Reg32Types.types in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_X">;
+}
+foreach vt = Reg64Types.types in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XY">;
+}
+foreach vt = Reg96Types.types in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XYZ">;
+}
+foreach vt = Reg128Types.types in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XYZW">;
+}
} // End HasMTBUFInsts.
let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
- defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
- defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
+ foreach vt = Reg16Types.types in {
+ defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, vt, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
+ }
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
@@ -2362,15 +2367,20 @@ let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
} // End HasUnpackedD16VMem,HasMTBUFInsts.
let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i16, "TBUFFER_LOAD_FORMAT_D16_X">;
+ foreach vt = Reg16Types.types in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, vt, "TBUFFER_LOAD_FORMAT_D16_X">;
+ }
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i16, "TBUFFER_LOAD_FORMAT_D16_XY">;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
- defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
+ foreach vt = [v2i16, v2f16, v2bf16] in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, vt, "TBUFFER_LOAD_FORMAT_D16_XY">;
+ }
+ foreach vt = [v4i16, v4f16] in {
+ defvar memoryVt = !cast<ValueType>("v3" # vt.ElementType);
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, vt, "TBUFFER_LOAD_FORMAT_D16_XYZ", memoryVt>;
+ }
+ foreach vt = [v4i16, v4f16, v4bf16] in {
+ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, vt, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
+ }
} // End HasPackedD16VMem,HasMTBUFInsts.
multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
@@ -2421,19 +2431,24 @@ multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
}
let OtherPredicates = [HasMTBUFInsts] in {
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
-defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
+foreach vt = Reg32Types.types in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_X">;
+}
+foreach vt = Reg64Types.types in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XY">;
+}
+foreach vt = Reg96Types.types in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XYZ">;
+}
+foreach vt = Reg128Types.types in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XYZW">;
+}
} // End HasMTBUFInsts.
let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
- defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
- defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
+ foreach vt = Reg16Types.types in {
+ defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
+ }
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
@@ -2441,15 +2456,20 @@ let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
} // End HasUnpackedD16VMem,HasMTBUFInsts.
let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i16, "TBUFFER_STORE_FORMAT_D16_X">;
+ foreach vt = Reg16Types.types in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_X">;
+ }
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i16, "TBUFFER_STORE_FORMAT_D16_XY">;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3i16>;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
- defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
+ foreach vt = [v2i16, v2f16, v2bf16] in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_XY">;
+ }
+ foreach vt = [v4i16, v4f16] in {
+ defvar memoryVt = !cast<ValueType>("v3" # vt.ElementType);
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_XYZ", memoryVt>;
+ }
+ foreach vt = [v4i16, v4f16, v4bf16] in {
+ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, vt, "TBUFFER_STORE_FORMAT_D16_XYZW">;
+ }
} // End HasPackedD16VMem,HasMTBUFInsts.
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d46c9d63f3404..c42a7d9b53d68 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -11453,7 +11453,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType().getSizeInBits() == 16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11480,7 +11480,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(1, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType().getSizeInBits() == 16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -12134,7 +12134,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType().getSizeInBits() == 16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12162,7 +12162,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType().getSizeInBits() == 16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
index ad06215c733a3..5efdf0c07c558 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
@@ -279,6 +279,104 @@ main_body:
ret i16 %elt
}
+; Bfloat element types must use the same D16 path as f16.
+define amdgpu_ps bfloat @tbuffer_load_bf16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x00,0xb4,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call bfloat @llvm.amdgcn.raw.ptr.tbuffer.load.bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ ret bfloat %data
+}
+
+define amdgpu_ps bfloat @tbuffer_load_v2bf16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_v2bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_xy v[0:1], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0xb4,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_mov_b32_e32 v0, v1 ; encoding: [0x01,0x03,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_v2bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_v2bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_v2bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xy v0, off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call <2 x bfloat> @llvm.amdgcn.raw.ptr.tbuffer.load.v2bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ %elt = extractelement <2 x bfloat> %data, i32 1
+ ret bfloat %elt
+}
+
+define amdgpu_ps bfloat @tbuffer_load_v4bf16(ptr addrspace(8) inreg %rsrc) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_load_v4bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM] ; encoding: [0x00,0x80,0xb5,0xe8,0x00,0x00,0x00,0x80]
+; PREGFX10-UNPACKED-NEXT: s_waitcnt vmcnt(0) ; encoding: [0x70,0x0f,0x8c,0xbf]
+; PREGFX10-UNPACKED-NEXT: v_mov_b32_e32 v0, v3 ; encoding: [0x03,0x03,0x00,0x7e]
+; PREGFX10-UNPACKED-NEXT: ; return to shader part epilog
+;
+; PREGFX10-PACKED-LABEL: tbuffer_load_v4bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
+; PREGFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; PREGFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; PREGFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX10-PACKED-LABEL: tbuffer_load_v4bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_load_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX10-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX10-PACKED-NEXT: ; return to shader part epilog
+;
+; GFX11-PACKED-LABEL: tbuffer_load_v4bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_load_d16_format_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
+; GFX11-PACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX11-PACKED-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-PACKED-NEXT: ; return to shader part epilog
+main_body:
+ %data = call <4 x bfloat> @llvm.amdgcn.raw.ptr.tbuffer.load.v4bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
+ %elt = extractelement <4 x bfloat> %data, i32 3
+ ret bfloat %elt
+}
+
declare half @llvm.amdgcn.raw.ptr.tbuffer.load.f16(ptr addrspace(8), i32, i32, i32, i32)
declare <2 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v2f16(ptr addrspace(8), i32, i32, i32, i32)
declare <3 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v3f16(ptr addrspace(8), i32, i32, i32, i32)
@@ -287,3 +385,6 @@ declare i16 @llvm.amdgcn.raw.ptr.tbuffer.load.i16(ptr addrspace(8), i32, i32, i3
declare <2 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v2i16(ptr addrspace(8), i32, i32, i32, i32)
declare <3 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v3i16(ptr addrspace(8), i32, i32, i32, i32)
declare <4 x i16> @llvm.amdgcn.raw.ptr.tbuffer.load.v4i16(ptr addrspace(8), i32, i32, i32, i32)
+declare bfloat @llvm.amdgcn.raw.ptr.tbuffer.load.bf16(ptr addrspace(8), i32, i32, i32, i32)
+declare <2 x bfloat> @llvm.amdgcn.raw.ptr.tbuffer.load.v2bf16(ptr addrspace(8), i32, i32, i32, i32)
+declare <4 x bfloat> @llvm.amdgcn.raw.ptr.tbuffer.load.v4bf16(ptr addrspace(8), i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
index 752b673ca6758..d17b6f4be0088 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
@@ -235,11 +235,96 @@ main_body:
ret void
}
+; Bfloat element types must use the same D16 path as f16.
+define amdgpu_ps void @tbuffer_store_bf16(ptr addrspace(8) inreg %rsrc, bfloat %data) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_store_bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: tbuffer_store_format_d16_x v0, off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-UNPACKED-NEXT: s_endpgm
+;
+; PREGFX10-PACKED-LABEL: tbuffer_store_bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-PACKED-NEXT: s_endpgm
+;
+; GFX10-PACKED-LABEL: tbuffer_store_bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_store_format_d16_x v0, off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED]
+; GFX10-PACKED-NEXT: s_endpgm
+;
+; GFX11-PACKED-LABEL: tbuffer_store_bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_store_d16_format_x v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM]
+; GFX11-PACKED-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.raw.ptr.tbuffer.store.bf16(bfloat %data, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @tbuffer_store_v2bf16(ptr addrspace(8) inreg %rsrc, <2 x bfloat> %data) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_store_v2bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; PREGFX10-UNPACKED-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; PREGFX10-UNPACKED-NEXT: tbuffer_store_format_d16_xy v[0:1], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-UNPACKED-NEXT: s_endpgm
+;
+; PREGFX10-PACKED-LABEL: tbuffer_store_v2bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_store_format_d16_xy v0, off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-PACKED-NEXT: s_endpgm
+;
+; GFX10-PACKED-LABEL: tbuffer_store_v2bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_store_format_d16_xy v0, off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED]
+; GFX10-PACKED-NEXT: s_endpgm
+;
+; GFX11-PACKED-LABEL: tbuffer_store_v2bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xy v0, off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM]
+; GFX11-PACKED-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.raw.ptr.tbuffer.store.v2bf16(<2 x bfloat> %data, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @tbuffer_store_v4bf16(ptr addrspace(8) inreg %rsrc, <4 x bfloat> %data) {
+; PREGFX10-UNPACKED-LABEL: tbuffer_store_v4bf16:
+; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
+; PREGFX10-UNPACKED-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; PREGFX10-UNPACKED-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; PREGFX10-UNPACKED-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; PREGFX10-UNPACKED-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; PREGFX10-UNPACKED-NEXT: tbuffer_store_format_d16_xyzw v[0:3], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-UNPACKED-NEXT: s_endpgm
+;
+; PREGFX10-PACKED-LABEL: tbuffer_store_v4bf16:
+; PREGFX10-PACKED: ; %bb.0: ; %main_body
+; PREGFX10-PACKED-NEXT: tbuffer_store_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_NUM_FORMAT_USCALED]
+; PREGFX10-PACKED-NEXT: s_endpgm
+;
+; GFX10-PACKED-LABEL: tbuffer_store_v4bf16:
+; GFX10-PACKED: ; %bb.0: ; %main_body
+; GFX10-PACKED-NEXT: tbuffer_store_format_d16_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_11_11_SSCALED]
+; GFX10-PACKED-NEXT: s_endpgm
+;
+; GFX11-PACKED-LABEL: tbuffer_store_v4bf16:
+; GFX11-PACKED: ; %bb.0: ; %main_body
+; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyzw v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM]
+; GFX11-PACKED-NEXT: s_endpgm
+main_body:
+ call void @llvm.amdgcn.raw.ptr.tbuffer.store.v4bf16(<4 x bfloat> %data, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0)
+ ret void
+}
+
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v3f16(<3 x half>, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v4f16(<4 x half>, ptr addrspace(8), i32, i32, i32, i32)
declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v3i16(<3 x i16>, ptr addrspace(8), i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.ptr.tbuffer.store.bf16(bfloat, ptr addrspace(8), i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v2bf16(<2 x bfloat>, ptr addrspace(8), i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.ptr.tbuffer.store.v4bf16(<4 x bfloat>, ptr addrspace(8), i32, i32, i32, i32)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11-PACKED-FAKE16: {{.*}}
; GFX11-PACKED-TRUE16: {{.*}}
>From 5c0be39ecf29fa21ca588c0b380fc7b5afa66814 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 8 Jun 2026 15:12:05 +0200
Subject: [PATCH 3/3] Address comments
---
llvm/lib/Target/AMDGPU/BUFInstructions.td | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 6753989636055..5faa478407c34 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2342,16 +2342,16 @@ multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
}
let OtherPredicates = [HasMTBUFInsts] in {
-foreach vt = Reg32Types.types in {
+foreach vt = [i32, f32] in {
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_X">;
}
-foreach vt = Reg64Types.types in {
+foreach vt = [v2i32, v2f32] in {
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XY">;
}
-foreach vt = Reg96Types.types in {
+foreach vt = [v3i32, v3f32] in {
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XYZ">;
}
-foreach vt = Reg128Types.types in {
+foreach vt = [v4i32, v4f32] in {
defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, vt, "TBUFFER_LOAD_FORMAT_XYZW">;
}
} // End HasMTBUFInsts.
@@ -2431,16 +2431,16 @@ multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
}
let OtherPredicates = [HasMTBUFInsts] in {
-foreach vt = Reg32Types.types in {
+foreach vt = [i32, f32] in {
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_X">;
}
-foreach vt = Reg64Types.types in {
+foreach vt = [v2i32, v2f32] in {
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XY">;
}
-foreach vt = Reg96Types.types in {
+foreach vt = [v3i32, v3f32] in {
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XYZ">;
}
-foreach vt = Reg128Types.types in {
+foreach vt = [v4i32, v4f32] in {
defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, vt, "TBUFFER_STORE_FORMAT_XYZW">;
}
} // End HasMTBUFInsts.
More information about the llvm-commits
mailing list