[llvm] [AMDGPU] Add custom lowering of llvm.convert.from.arbitrary.fp for FP8 (PR #194144)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Sun May 3 14:33:34 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/194144
>From e016d8c6c63280896f30ddd3d3b3c099df4fb252 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 25 Apr 2026 15:18:23 +0200
Subject: [PATCH 1/5] [AMDGPU] Add custom lowering of
llvm.convert.from.arbitrary.fp for FP8
Map conversions from FP8 source formats to v_cvt_{,pk_}f32_{fp8,bf8}
HW instructions.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 82 +++++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 342 ++++++++++++++++++
3 files changed, 425 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8bb072a9626a9..37a8a008e0552 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -21,6 +21,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
@@ -985,6 +986,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
+ if (Subtarget->hasFP8ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
+ {MVT::f32, MVT::v2f32, MVT::v4f32}, Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::v2i8, MVT::v4i8},
+ Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -7496,6 +7504,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerExternalSymbol(Op, DAG);
case ISD::INTRINSIC_WO_CHAIN:
return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+ case ISD::CONVERT_FROM_ARBITRARY_FP:
+ return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
case ISD::INTRINSIC_W_CHAIN:
return LowerINTRINSIC_W_CHAIN(Op, DAG);
case ISD::INTRINSIC_VOID:
@@ -10569,6 +10579,78 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT DstVT = Op.getValueType();
+ if (DstVT != MVT::f32 && DstVT != MVT::v2f32 && DstVT != MVT::v4f32)
+ return SDValue();
+
+ bool IsBF8 = false;
+ switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
+ case APFloatBase::S_Float8E4M3FN:
+ IsBF8 = false;
+ break;
+ case APFloatBase::S_Float8E5M2:
+ IsBF8 = true;
+ break;
+ default:
+ return SDValue();
+ }
+
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // Defer constant inputs to generic bit-twiddling expansion.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Src))
+ return SDValue();
+
+ // Pack vector inputs as hw instruction is packed.
+ // FIXME: note, packing loses lane-wise poison. Should we do something about
+ // it?
+ auto PackBytes = [&](unsigned NumBytes, unsigned FirstLane) {
+ EVT EltVT = Src.getValueType().getVectorElementType();
+ SDValue PackedI32;
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ SDValue Elt = DAG.getExtractVectorElt(SL, EltVT, Src, FirstLane + I);
+ SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
+ Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
+ DAG.getConstant(0xFF, SL, MVT::i32));
+ if (I != 0)
+ Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
+ DAG.getConstant(I * 8, SL, MVT::i32));
+ PackedI32 =
+ I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
+ }
+ return PackedI32;
+ };
+
+ auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
+ unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
+ : Intrinsic::amdgcn_cvt_pk_f32_fp8;
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI32,
+ DAG.getTargetConstant(WordSel, SL, MVT::i1));
+ };
+
+ if (DstVT == MVT::f32) {
+ unsigned IntrID =
+ IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
+ DAG.getTargetConstant(0, SL, MVT::i32));
+ }
+
+ if (DstVT == MVT::v2f32)
+ return EmitPk(PackBytes(2, 0), 0);
+
+ SDValue PackedI32 = PackBytes(4, 0);
+ SDValue Lo = EmitPk(PackedI32, 0);
+ SDValue Hi = EmitPk(PackedI32, 1);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+}
+
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index e37bd938dc35d..e19e3d580e349 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -103,6 +103,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
new file mode 100644
index 0000000000000..87ea18db6feb8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -0,0 +1,342 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+
+; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
+
+declare float @llvm.convert.from.arbitrary.fp.f32.i8(i8, metadata)
+declare <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8>, metadata)
+declare <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8>, metadata)
+
+; Scalar Float8E5M2
+define float @from_bf8_dynamic(i8 %x) {
+; GFX942-LABEL: from_bf8_dynamic:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_cvt_f32_bf8_sdwa v0, v0 src0_sel:BYTE_0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: from_bf8_dynamic:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: from_bf8_dynamic:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: from_bf8_dynamic:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E5M2")
+ ret float %r
+}
+
+; Scalar Float8E4M3FN
+define float @from_fp8_dynamic(i8 %x) {
+; GFX942-LABEL: from_fp8_dynamic:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_cvt_f32_fp8_sdwa v0, v0 src0_sel:BYTE_0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: from_fp8_dynamic:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: from_fp8_dynamic:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: from_fp8_dynamic:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E4M3FN")
+ ret float %r
+}
+
+; v2f32
+define <2 x float> @v2_from_bf8(<2 x i8> %x) {
+; GFX942-LABEL: v2_from_bf8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v2_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v2_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v2_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
+ ret <2 x float> %r
+}
+
+define <2 x float> @v2_from_fp8(<2 x i8> %x) {
+; GFX942-LABEL: v2_from_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v2_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v2_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v2_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
+ ret <2 x float> %r
+}
+
+; v4f32
+define <4 x float> @v4_from_bf8(<4 x i8> %x) {
+; GFX942-LABEL: v4_from_bf8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX942-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v4_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v4_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v4_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
+ ret <4 x float> %r
+}
+
+define <4 x float> @v4_from_fp8(<4 x i8> %x) {
+; GFX942-LABEL: v4_from_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX942-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v4_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v4_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v4_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
+ ret <4 x float> %r
+}
+
+; Fall through to the generic expansion in case of constant inputs so they fold..
+define float @const_fp8() {
+; GFX942-LABEL: const_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: const_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: const_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: const_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 56, metadata !"Float8E4M3FN")
+ ret float %r
+}
+
+define <2 x float> @const_v2_fp8() {
+; GFX942-LABEL: const_v2_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX942-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: const_v2_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX950-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: const_v2_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: const_v2_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> <i8 56, i8 64>, metadata !"Float8E4M3FN")
+ ret <2 x float> %r
+}
>From 38b7e5321bb88091ca7fb1cf63526e5c820572e4 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 26 Apr 2026 23:54:57 +0200
Subject: [PATCH 2/5] add fp16 lowering, address comments and do a bit of
refactoring for next
conversions
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 150 ++++++++++++------
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 84 ++++++++++
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 12 +-
4 files changed, 192 insertions(+), 58 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 37a8a008e0552..9525fe22e853b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -993,6 +993,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
Custom);
}
+ if (Subtarget->hasGFX1250Insts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
+ {MVT::f16, MVT::v2f16, MVT::v4f16}, Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -10579,51 +10584,42 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
-SDValue
-SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
- SelectionDAG &DAG) const {
+// Pack vector inputs as hw instruction is packed.
+// FIXME: note, packing loses lane-wise poison. Should we do something about
+// it?
+SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
+ SDValue Src, unsigned NumBytes,
+ unsigned FirstLane) {
+ EVT SrcEltVT = Src.getValueType().getVectorElementType();
+ SDValue PackedI32;
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ SDValue Elt = DAG.getExtractVectorElt(SL, SrcEltVT, Src, FirstLane + I);
+ SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
+ Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
+ DAG.getConstant(0xFF, SL, MVT::i32));
+ if (I != 0)
+ Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
+ DAG.getConstant(I * 8, SL, MVT::i32));
+ PackedI32 =
+ I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
+ }
+ return PackedI32;
+}
+
+SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
EVT DstVT = Op.getValueType();
- if (DstVT != MVT::f32 && DstVT != MVT::v2f32 && DstVT != MVT::v4f32)
- return SDValue();
-
- bool IsBF8 = false;
- switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
- case APFloatBase::S_Float8E4M3FN:
- IsBF8 = false;
- break;
- case APFloatBase::S_Float8E5M2:
- IsBF8 = true;
- break;
- default:
- return SDValue();
- }
-
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- // Defer constant inputs to generic bit-twiddling expansion.
- if (DAG.isConstantIntBuildVectorOrConstantInt(Src))
- return SDValue();
-
- // Pack vector inputs as hw instruction is packed.
- // FIXME: note, packing loses lane-wise poison. Should we do something about
- // it?
- auto PackBytes = [&](unsigned NumBytes, unsigned FirstLane) {
- EVT EltVT = Src.getValueType().getVectorElementType();
- SDValue PackedI32;
- for (unsigned I = 0; I != NumBytes; ++I) {
- SDValue Elt = DAG.getExtractVectorElt(SL, EltVT, Src, FirstLane + I);
- SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
- Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
- DAG.getConstant(0xFF, SL, MVT::i32));
- if (I != 0)
- Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
- DAG.getConstant(I * 8, SL, MVT::i32));
- PackedI32 =
- I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
- }
- return PackedI32;
- };
+ if (DstVT == MVT::f32) {
+ unsigned IntrID =
+ IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
+ DAG.getTargetConstant(0, SL, MVT::i32));
+ }
auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
@@ -10633,22 +10629,76 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
DAG.getTargetConstant(WordSel, SL, MVT::i1));
};
- if (DstVT == MVT::f32) {
+ if (DstVT == MVT::v2f32)
+ return EmitPk(packBytesToI32(DAG, SL, Src, 2, 0), 0);
+
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 4, 0);
+ SDValue Lo = EmitPk(PackedI32, 0);
+ SDValue Hi = EmitPk(PackedI32, 1);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+}
+
+SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasGFX1250Insts() &&
+ "unscaled fp8/bf8 -> f16 requires gfx1250+");
+ EVT DstVT = Op.getValueType();
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // amdgcn_cvt_f16_{fp8,bf8}(i32 src, byte_sel) -> f16.
+ // amdgcn_cvt_pk_f16_{fp8,bf8}(i16 src) -> v2f16.
+ if (!DstVT.isVector()) {
unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ IsBF8 ? Intrinsic::amdgcn_cvt_f16_bf8 : Intrinsic::amdgcn_cvt_f16_fp8;
SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f16,
DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
DAG.getTargetConstant(0, SL, MVT::i32));
}
- if (DstVT == MVT::v2f32)
- return EmitPk(PackBytes(2, 0), 0);
+ unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
+ : Intrinsic::amdgcn_cvt_pk_f16_fp8;
+ SDValue IntrConst = DAG.getTargetConstant(IntrID, SL, MVT::i32);
+ auto EmitPk = [&](unsigned FirstLane) {
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 2, FirstLane);
+ SDValue PackedI16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, PackedI32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f16, IntrConst,
+ PackedI16);
+ };
+ if (DstVT.getVectorNumElements() == 2)
+ return EmitPk(0);
+ SDValue Lo = EmitPk(0);
+ SDValue Hi = EmitPk(2);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, DstVT, Lo, Hi);
+}
- SDValue PackedI32 = PackBytes(4, 0);
- SDValue Lo = EmitPk(PackedI32, 0);
- SDValue Hi = EmitPk(PackedI32, 1);
- return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ bool IsBF8 = false;
+ switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
+ case APFloatBase::S_Float8E4M3FN:
+ IsBF8 = false;
+ break;
+ case APFloatBase::S_Float8E5M2:
+ IsBF8 = true;
+ break;
+ default:
+ return SDValue();
+ }
+
+ // Defer constant inputs to generic bit-twiddling expansion.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(0)))
+ return SDValue();
+
+ EVT DstVT = Op.getValueType();
+ EVT EltVT = DstVT.isVector() ? DstVT.getVectorElementType() : DstVT;
+ if (EltVT == MVT::f16)
+ return lowerFromFP8ToF16(Op, IsBF8, DAG);
+ if (DstVT == MVT::f32 || DstVT == MVT::v2f32 || DstVT == MVT::v4f32)
+ return lowerFromFP8ToF32(Op, IsBF8, DAG);
+ return SDValue();
}
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index e19e3d580e349..c0ca80ab23177 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -104,6 +104,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8ToF32(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8ToF16(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
+ static SDValue packBytesToI32(SelectionDAG &DAG, const SDLoc &SL, SDValue Src,
+ unsigned NumBytes, unsigned FirstLane);
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
new file mode 100644
index 0000000000000..752d037cb9691
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+
+; llvm.convert.from.arbitrary.fp from FP8/BF8 producing f16 result types,
+; lowered via the gfx1250+ unscaled v_cvt_{pk_}f16_{fp8,bf8} instructions.
+
+; Scalar Float8E4M3FN
+define half @from_fp8_f16_s(i8 %x) {
+; GFX1250-LABEL: from_fp8_f16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E4M3FN")
+ ret half %r
+}
+
+; Scalar Float8E5M2
+define half @from_bf8_f16_s(i8 %x) {
+; GFX1250-LABEL: from_bf8_f16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E5M2")
+ ret half %r
+}
+
+; Vector Float8E4M3FN
+define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
+ ret <2 x half> %r
+}
+
+; Vector Float8E5M2
+define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
+; GFX1250-LABEL: from_bf8_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
+ ret <2 x half> %r
+}
+
+define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v4f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v1, v3, v2, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
+ ret <4 x half> %r
+}
+
+; Constant input bypasses HW-specific path.
+define half @from_fp8_f16_const() {
+; GFX1250-LABEL: from_fp8_f16_const:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 56, metadata !"Float8E4M3FN")
+ ret half %r
+}
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index 87ea18db6feb8..2597e607a5a1f 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -1,15 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
-declare float @llvm.convert.from.arbitrary.fp.f32.i8(i8, metadata)
-declare <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8>, metadata)
-declare <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8>, metadata)
-
; Scalar Float8E5M2
define float @from_bf8_dynamic(i8 %x) {
; GFX942-LABEL: from_bf8_dynamic:
>From 82273bb04779e4d25d51a4b761f14fbb788c8c0c Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 30 Apr 2026 15:08:57 -0500
Subject: [PATCH 3/5] Guard FP8 custom lowering with hasOCPFP8ConversionInsts
---
.../include/llvm/Target/TargetSelectionDAG.td | 5 +
llvm/lib/Target/AMDGPU/AMDGPU.td | 16 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 86 +++---
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 9 +
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 38 ++-
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 30 ++-
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 253 +++++++++---------
7 files changed, 256 insertions(+), 181 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 573342846b4cf..e53ff4d974e2a 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -175,6 +175,9 @@ def SDIsFPClassOp : SDTypeProfile<1, 2, [ // is_fpclass
def SDTIntToFPOp : SDTypeProfile<1, 1, [ // [su]int_to_fp
SDTCisFP<0>, SDTCisInt<1>, SDTCisSameNumEltsAs<0, 1>
]>;
+def SDTConvertFromArbitraryFPOp : SDTypeProfile<1, 2, [
+ SDTCisFP<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisSameNumEltsAs<0, 1>
+]>;
def SDTFPToIntOp : SDTypeProfile<1, 1, [ // fp_to_[su]int
SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>
]>;
@@ -637,6 +640,8 @@ def is_fpclass : SDNode<"ISD::IS_FPCLASS" , SDIsFPClassOp>;
def sint_to_fp : SDNode<"ISD::SINT_TO_FP" , SDTIntToFPOp>;
def uint_to_fp : SDNode<"ISD::UINT_TO_FP" , SDTIntToFPOp>;
+def convert_from_arbitrary_fp : SDNode<"ISD::CONVERT_FROM_ARBITRARY_FP",
+ SDTConvertFromArbitraryFPOp>;
def fp_to_sint : SDNode<"ISD::FP_TO_SINT" , SDTFPToIntOp>;
def fp_to_uint : SDNode<"ISD::FP_TO_UINT" , SDTFPToIntOp>;
def fp_to_sint_sat : SDNode<"ISD::FP_TO_SINT_SAT" , SDTFPToIntSatOp>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 25fc64d178858..7d6b968845731 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -795,6 +795,15 @@ defm FP8ConversionInsts : AMDGPUSubtargetFeature<"fp8-conversion-insts",
"Has fp8 and bf8 conversion instructions"
>;
+// Note: it doesn't include FNUZ dataformat, therefore gfx942 is excluded.
+defm OCPFP8ConversionInsts : AMDGPUSubtargetFeature<"ocp-fp8-conversion-insts",
+ "Has OCP fp8 (E4M3FN, E5M2) conversion instructions"
+>;
+
+defm FP8F16ConversionInsts : AMDGPUSubtargetFeature<"fp8-f16-conversion-insts",
+ "Has fp8 and bf8 conversion to f16 instructions"
+>;
+
defm FP8E5M3Insts : AMDGPUSubtargetFeature<"fp8e5m3-insts",
"Has fp8 e5m3 format support"
>;
@@ -1762,6 +1771,7 @@ def FeatureISAVersion9_5_Common : FeatureSet<
[FeatureAddressableLocalMemorySize163840,
FeatureFP8Insts,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureGFX950Insts,
FeaturePrngInst,
FeatureBF16ConversionInsts,
@@ -1969,6 +1979,7 @@ def FeatureISAVersion11_7_Common : FeatureSet<
FeatureSALUFloatInsts,
FeatureDPPSrc1SGPR,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureDot11Insts,
FeatureWMMA128bInsts,
FeatureSWMMACGfx1200Insts,
@@ -2005,6 +2016,7 @@ def FeatureISAVersion12 : FeatureSet<
FeatureImageInsts,
FeatureExtendedImageInsts,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureWMMA128bInsts,
FeatureSWMMACGfx1200Insts,
FeatureIEEEMinimumMaximumInsts,
@@ -2060,6 +2072,8 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureAtomicBufferPkAddBF16Inst,
FeatureFlatAtomicFaddF32Inst,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
+ FeatureFP8F16ConversionInsts,
FeatureFP8E5M3Insts,
FeaturePackedTID,
FeatureVcmpxPermlaneHazard,
@@ -2173,6 +2187,8 @@ def FeatureISAVersion13 : FeatureSet<
FeatureAtomicBufferPkAddBF16Inst,
FeatureFlatAtomicFaddF32Inst,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
+ FeatureFP8F16ConversionInsts,
FeaturePackedTID,
FeatureVcmpxPermlaneHazard,
FeatureSALUFloatInsts,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9525fe22e853b..81e84bacde106 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -986,14 +986,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
- if (Subtarget->hasFP8ConversionInsts()) {
+ if (Subtarget->hasOCPFP8ConversionInsts()) {
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
{MVT::f32, MVT::v2f32, MVT::v4f32}, Custom);
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::v2i8, MVT::v4i8},
Custom);
}
- if (Subtarget->hasGFX1250Insts()) {
+ if (Subtarget->hasFP8F16ConversionInsts()) {
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
{MVT::f16, MVT::v2f16, MVT::v4f16}, Custom);
}
@@ -10584,26 +10584,32 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
-// Pack vector inputs as hw instruction is packed.
-// FIXME: note, packing loses lane-wise poison. Should we do something about
-// it?
+// Pack a vector of i8 lanes into i32 via bitcast.
+// FIXME: packing loses lane-wise poison. Should we do something about it?
SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
SDValue Src, unsigned NumBytes,
unsigned FirstLane) {
- EVT SrcEltVT = Src.getValueType().getVectorElementType();
- SDValue PackedI32;
- for (unsigned I = 0; I != NumBytes; ++I) {
- SDValue Elt = DAG.getExtractVectorElt(SL, SrcEltVT, Src, FirstLane + I);
- SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
- Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
- DAG.getConstant(0xFF, SL, MVT::i32));
- if (I != 0)
- Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
- DAG.getConstant(I * 8, SL, MVT::i32));
- PackedI32 =
- I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
- }
- return PackedI32;
+ EVT SrcVT = Src.getValueType();
+ assert(SrcVT.isVector() &&
+ SrcVT.getVectorElementType() == MVT::i8 &&
+ "packBytesToI32 expects a v*i8 source");
+ unsigned SrcBytes = SrcVT.getVectorNumElements();
+ assert((NumBytes == 2 || NumBytes == 4) && "expected 2 or 4 bytes");
+
+ if (NumBytes == 4) {
+ assert(FirstLane == 0 && "v4i8 -> i32 takes the full vector");
+ return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Src);
+ }
+
+ // Bitcast a vector to the same-width integer, optionally shift down to the
+ // requested slice, truncate to i16, and zext to i32.
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcBytes * 8);
+ SDValue AsInt = DAG.getNode(ISD::BITCAST, SL, IntVT, Src);
+ if (FirstLane != 0)
+ AsInt = DAG.getNode(ISD::SRL, SL, IntVT, AsInt,
+ DAG.getConstant(FirstLane * 8, SL, IntVT));
+ SDValue I16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, AsInt);
+ return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, I16);
}
SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
@@ -10612,14 +10618,7 @@ SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- if (DstVT == MVT::f32) {
- unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
- SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
- DAG.getTargetConstant(0, SL, MVT::i32));
- }
+ assert(DstVT.isVector() && "Scalar f32 should be selected by TableGen");
auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
@@ -10640,22 +10639,14 @@ SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
SelectionDAG &DAG) const {
- assert(Subtarget->hasGFX1250Insts() &&
- "unscaled fp8/bf8 -> f16 requires gfx1250+");
+ assert(Subtarget->hasFP8F16ConversionInsts() &&
+ "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
EVT DstVT = Op.getValueType();
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- // amdgcn_cvt_f16_{fp8,bf8}(i32 src, byte_sel) -> f16.
- // amdgcn_cvt_pk_f16_{fp8,bf8}(i16 src) -> v2f16.
- if (!DstVT.isVector()) {
- unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f16_bf8 : Intrinsic::amdgcn_cvt_f16_fp8;
- SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f16,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
- DAG.getTargetConstant(0, SL, MVT::i32));
- }
+ // Scalar f16 is handled by TableGen patterns.
+ assert(DstVT.isVector() && "Scalar f16 should be selected by TableGen");
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
: Intrinsic::amdgcn_cvt_pk_f16_fp8;
@@ -10676,6 +10667,8 @@ SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
SDValue
SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
SelectionDAG &DAG) const {
+ // Only handle OCP FP8 formats (E4M3FN, E5M2). FNUZ formats that are supported
+ // by gfx942 fall through to the generic expansion.
bool IsBF8 = false;
switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
case APFloatBase::S_Float8E4M3FN:
@@ -10693,10 +10686,21 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
return SDValue();
EVT DstVT = Op.getValueType();
- EVT EltVT = DstVT.isVector() ? DstVT.getVectorElementType() : DstVT;
+ if (!DstVT.isVector()) {
+ SDValue Src = Op.getOperand(0);
+ if (Src.getValueType() != MVT::i32) {
+ SDLoc SL(Op);
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, SL, DstVT, SrcI32,
+ Op.getOperand(1));
+ }
+ return Op;
+ }
+
+ EVT EltVT = DstVT.getVectorElementType();
if (EltVT == MVT::f16)
return lowerFromFP8ToF16(Op, IsBF8, DAG);
- if (DstVT == MVT::f32 || DstVT == MVT::v2f32 || DstVT == MVT::v4f32)
+ if (EltVT == MVT::f32)
return lowerFromFP8ToF32(Op, IsBF8, DAG);
return SDValue();
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index ef6d0f22a2f3e..a24624f5e25df 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -911,6 +911,15 @@ def RtzRoundMode : TImmLeaf<i32, [{
return Imm == (int)RoundingMode::TowardZero;
}]>;
+// llvm.convert.from.arbitrary.fp format-selector predicates.
+// Only OCP formats are matched here; FNUZ formats use the generic expansion.
+def fp8_sem : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E4M3FN;
+}]>;
+def bf8_sem : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E5M2;
+}]>;
+
def VOP3PModsNeg : SDNodeXForm<timm, [{
unsigned Mods = SISrcMods::OP_SEL_1;
if (N->getZExtValue())
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 9955d60fb2849..6c675c32f69c2 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -726,6 +726,15 @@ foreach Index = [1, 2, 3] in {
}
} // End OtherPredicates = [HasSDWA]
+// llvm.convert.from.arbitrary.fp i8 -> f32 scalar patterns.
+let OtherPredicates = [NotHasCvtFP8VOP1Bug, HasSDWA] in {
+ // gfx950 has OCP fp8 hardware with SDWA available.
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_e32 $src)>;
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 bf8_sem))),
+ (V_CVT_F32_BF8_e32 $src)>;
+}
+
} // End SubtargetPredicate = HasFP8ConversionInsts
class Cvt_PK_F32_F8_Pat<SDPatternOperator node, int index,
@@ -811,6 +820,17 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
}
let SubtargetPredicate = isGFX11Plus in
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
+
+ // llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns.
+ let SubtargetPredicate = isGFX11PlusNot12_50 in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
+ let SubtargetPredicate = isGFX125xOnly in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
+ let SubtargetPredicate = isGFX11Plus in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
}
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
@@ -837,7 +857,7 @@ def VOPProfile_CVT_PK_F16_F8_true16 : VOP3_Profile_True16<VOPProfile_CVT_PK_F16_
def VOPProfile_CVT_PK_F16_F8_fake16 : VOP3_Profile_Fake16<VOPProfile_CVT_PK_F16_F8>;
}
-let SubtargetPredicate = isGFX1250Plus in {
+let SubtargetPredicate = HasFP8F16ConversionInsts in {
let mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
defm V_CVT_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_fp8",
V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
@@ -860,6 +880,22 @@ let SubtargetPredicate = isGFX1250Plus in {
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
}
+ // llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+ let True16Predicate = UseRealTrue16Insts in {
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F16_FP8_t16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F16_BF8_t16_e64 0, $src0, 0)>;
+ }
+ let True16Predicate = UseFakeTrue16Insts in {
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F16_FP8_fake16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F16_BF8_fake16_e64 0, $src0, 0)>;
+ }
+} // End SubtargetPredicate = HasFP8F16ConversionInsts
+
+let SubtargetPredicate = isGFX1250Plus in {
defm V_SAT_PK4_I4_I8 : VOP1Inst_t16<"v_sat_pk4_i4_i8", VOP1_I16_I32, int_amdgcn_sat_pk4_i4_i8>;
defm V_SAT_PK4_U4_U8 : VOP1Inst_t16<"v_sat_pk4_u4_u8", VOP1_I16_I32, int_amdgcn_sat_pk4_u4_u8>;
} // End SubtargetPredicate = isGFX1250Plus
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index 752d037cb9691..7d63187038f61 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -34,8 +34,9 @@ define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
@@ -48,21 +49,38 @@ define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
ret <2 x half> %r
}
+; v3f16
+define <3 x half> @from_fp8_v3f16(<3 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v3f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
+ ret <3 x half> %r
+}
+
define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
; GFX1250-LABEL: from_fp8_v4f16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_perm_b32 v1, v3, v2, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index 2597e607a5a1f..ba129f08406fb 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -1,19 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
-; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
+; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets with OCP FP8.
; Scalar Float8E5M2
define float @from_bf8_dynamic(i8 %x) {
-; GFX942-LABEL: from_bf8_dynamic:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_cvt_f32_bf8_sdwa v0, v0 src0_sel:BYTE_0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: from_bf8_dynamic:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -38,12 +31,6 @@ define float @from_bf8_dynamic(i8 %x) {
; Scalar Float8E4M3FN
define float @from_fp8_dynamic(i8 %x) {
-; GFX942-LABEL: from_fp8_dynamic:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_cvt_f32_fp8_sdwa v0, v0 src0_sel:BYTE_0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: from_fp8_dynamic:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -68,30 +55,19 @@ define float @from_fp8_dynamic(i8 %x) {
; v2f32
define <2 x float> @v2_from_bf8(<2 x i8> %x) {
-; GFX942-LABEL: v2_from_bf8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v2_from_bf8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v2_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
@@ -99,9 +75,8 @@ define <2 x float> @v2_from_bf8(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
@@ -109,30 +84,19 @@ define <2 x float> @v2_from_bf8(<2 x i8> %x) {
}
define <2 x float> @v2_from_fp8(<2 x i8> %x) {
-; GFX942-LABEL: v2_from_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v2_from_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v2_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
@@ -140,9 +104,8 @@ define <2 x float> @v2_from_fp8(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
@@ -151,30 +114,13 @@ define <2 x float> @v2_from_fp8(<2 x i8> %x) {
; v4f32
define <4 x float> @v4_from_bf8(<4 x i8> %x) {
-; GFX942-LABEL: v4_from_bf8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
-; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX942-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v4_from_bf8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
+; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
; GFX950-NEXT: s_setpc_b64 s[30:31]
@@ -182,14 +128,10 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX1170-LABEL: v4_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1170-NEXT: s_setpc_b64 s[30:31]
@@ -198,14 +140,10 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -214,30 +152,13 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
}
define <4 x float> @v4_from_fp8(<4 x i8> %x) {
-; GFX942-LABEL: v4_from_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
-; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX942-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v4_from_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
+; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
; GFX950-NEXT: s_setpc_b64 s[30:31]
@@ -245,14 +166,10 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX1170-LABEL: v4_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1170-NEXT: s_setpc_b64 s[30:31]
@@ -261,14 +178,10 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -276,14 +189,95 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
ret <4 x float> %r
}
+; v3f32
+define <3 x float> @v3_from_bf8(<3 x i8> %x) {
+; GFX950-LABEL: v3_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v3_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v3_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E5M2")
+ ret <3 x float> %r
+}
+
+define <3 x float> @v3_from_fp8(<3 x i8> %x) {
+; GFX950-LABEL: v3_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v3_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v3_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
+ ret <3 x float> %r
+}
+
; Fall through to the generic expansion in case of constant inputs so they fold..
define float @const_fp8() {
-; GFX942-LABEL: const_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: const_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -307,13 +301,6 @@ define float @const_fp8() {
}
define <2 x float> @const_v2_fp8() {
-; GFX942-LABEL: const_v2_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: const_v2_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
>From fce9b15f84109cfd973e64d158dd902044d4f38e Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 1 May 2026 01:03:47 +0200
Subject: [PATCH 4/5] format
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 91b3c112f8348..6fb8aafa328f8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10590,8 +10590,7 @@ SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
SDValue Src, unsigned NumBytes,
unsigned FirstLane) {
EVT SrcVT = Src.getValueType();
- assert(SrcVT.isVector() &&
- SrcVT.getVectorElementType() == MVT::i8 &&
+ assert(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i8 &&
"packBytesToI32 expects a v*i8 source");
unsigned SrcBytes = SrcVT.getVectorNumElements();
assert((NumBytes == 2 || NumBytes == 4) && "expected 2 or 4 bytes");
>From c5e585fa2b8bb400edfe7be1ea25b58f67bd7e7a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 2 May 2026 12:44:34 +0200
Subject: [PATCH 5/5] Apply comments, move to TargetSelectionDAG.td
---
.../include/llvm/Target/TargetSelectionDAG.td | 9 ++++
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 9 ----
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 46 ++++++++++---------
.../GlobalISelEmitter/GlobalISelEmitter.td | 10 +++-
4 files changed, 43 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index e53ff4d974e2a..00c1b29227a61 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -1264,6 +1264,15 @@ class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
SDNode ImmNode = timm> : ImmLeaf<vt, pred, xform, ImmNode>;
+// llvm.convert.from.arbitrary.fp format-selector predicates.
+// Only OCP formats are matched here; FNUZ formats use the generic expansion.
+def Float8E4M3FN : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E4M3FN;
+}]>;
+def Float8E5M2 : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E5M2;
+}]>;
+
// An ImmLeaf where the `Imm` in the C++ predicate is an `APInt`.
//
// This is useful when you need to zero-extend the immediate instead of
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 8b01adb7ef50a..7c3aff9c9f47f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -911,15 +911,6 @@ def RtzRoundMode : TImmLeaf<i32, [{
return Imm == (int)RoundingMode::TowardZero;
}]>;
-// llvm.convert.from.arbitrary.fp format-selector predicates.
-// Only OCP formats are matched here; FNUZ formats use the generic expansion.
-def fp8_sem : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E4M3FN;
-}]>;
-def bf8_sem : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E5M2;
-}]>;
-
def VOP3PModsNeg : SDNodeXForm<timm, [{
unsigned Mods = SISrcMods::OP_SEL_1;
if (N->getZExtValue())
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 6c675c32f69c2..5725b47a3598e 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -727,11 +727,11 @@ foreach Index = [1, 2, 3] in {
} // End OtherPredicates = [HasSDWA]
// llvm.convert.from.arbitrary.fp i8 -> f32 scalar patterns.
-let OtherPredicates = [NotHasCvtFP8VOP1Bug, HasSDWA] in {
- // gfx950 has OCP fp8 hardware with SDWA available.
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 fp8_sem))),
+// gfx950 has SDWA and uses the e32 form.
+let OtherPredicates = [HasOCPFP8ConversionInsts, HasSDWA] in {
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
(V_CVT_F32_FP8_e32 $src)>;
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 bf8_sem))),
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
(V_CVT_F32_BF8_e32 $src)>;
}
@@ -820,19 +820,20 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
}
let SubtargetPredicate = isGFX11Plus in
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
-
- // llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns.
- let SubtargetPredicate = isGFX11PlusNot12_50 in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
- (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
- let SubtargetPredicate = isGFX125xOnly in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
- (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
- let SubtargetPredicate = isGFX11Plus in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
- (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
}
+// llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns for GFX11+.
+// GFX11+ doesn't have SDWA, so must use OP_SEL form.
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus, isNotGFX1250Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
+ (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX1250Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
+ (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
+ (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
+
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
VOP1_Pseudo inst_e32, VOP3_Pseudo inst_e64> : GCNPat<
(v2f32 (node i32:$src, index)),
@@ -880,20 +881,23 @@ let SubtargetPredicate = HasFP8F16ConversionInsts in {
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
}
- // llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+} // End SubtargetPredicate = HasFP8F16ConversionInsts
+
+// llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+let OtherPredicates = [HasOCPFP8ConversionInsts, HasFP8F16ConversionInsts] in {
let True16Predicate = UseRealTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
(V_CVT_F16_FP8_t16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
(V_CVT_F16_BF8_t16_e64 0, $src0, 0)>;
}
let True16Predicate = UseFakeTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
(V_CVT_F16_FP8_fake16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
(V_CVT_F16_BF8_fake16_e64 0, $src0, 0)>;
}
-} // End SubtargetPredicate = HasFP8F16ConversionInsts
+}
let SubtargetPredicate = isGFX1250Plus in {
defm V_SAT_PK4_I4_I8 : VOP1Inst_t16<"v_sat_pk4_i4_i8", VOP1_I16_I32, int_amdgcn_sat_pk4_i4_i8>;
diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
index d09034fd53eca..6191170d0f031 100644
--- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
@@ -201,12 +201,20 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
// CHECK-LABEL: // PatFrag predicates.
// CHECK-NEXT: enum {
-// CHECK-NEXT: GICXXPred_I64_Predicate_cimm8 = GICXXPred_Invalid + 1,
+// CHECK-NEXT: GICXXPred_I64_Predicate_Float8E4M3FN = GICXXPred_Invalid + 1,
+// CHECK-NEXT: GICXXPred_I64_Predicate_Float8E5M2,
+// CHECK-NEXT: GICXXPred_I64_Predicate_cimm8,
// CHECK-NEXT: GICXXPred_I64_Predicate_simm8,
// CHECK-NEXT: };
// CHECK-NEXT: bool MyTargetInstructionSelector::testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const {
// CHECK-NEXT: switch (PredicateID) {
+// CHECK-NEXT: case GICXXPred_I64_Predicate_Float8E4M3FN: {
+// CHECK-NEXT: return Imm == APFloatBase::S_Float8E4M3FN;
+// CHECK-NEXT: }
+// CHECK-NEXT: case GICXXPred_I64_Predicate_Float8E5M2: {
+// CHECK-NEXT: return Imm == APFloatBase::S_Float8E5M2;
+// CHECK-NEXT: }
// CHECK-NEXT: case GICXXPred_I64_Predicate_cimm8: {
// CHECK-NEXT: return isInt<8>(Imm);
// CHECK-NEXT: }
More information about the llvm-commits
mailing list