[llvm] [AMDGPU] Add custom lowering of llvm.convert.from.arbitrary.fp for FP8 (PR #194144)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 5 15:31:51 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/194144
>From e016d8c6c63280896f30ddd3d3b3c099df4fb252 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 25 Apr 2026 15:18:23 +0200
Subject: [PATCH 01/12] [AMDGPU] Add custom lowering of
llvm.convert.from.arbitrary.fp for FP8
Map conversions from FP8 source formats to v_cvt_{,pk_}f32_{fp8,bf8}
HW instructions.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 82 +++++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 342 ++++++++++++++++++
3 files changed, 425 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8bb072a9626a9..37a8a008e0552 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -21,6 +21,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
@@ -985,6 +986,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
+ if (Subtarget->hasFP8ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
+ {MVT::f32, MVT::v2f32, MVT::v4f32}, Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::v2i8, MVT::v4i8},
+ Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -7496,6 +7504,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerExternalSymbol(Op, DAG);
case ISD::INTRINSIC_WO_CHAIN:
return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+ case ISD::CONVERT_FROM_ARBITRARY_FP:
+ return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
case ISD::INTRINSIC_W_CHAIN:
return LowerINTRINSIC_W_CHAIN(Op, DAG);
case ISD::INTRINSIC_VOID:
@@ -10569,6 +10579,78 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT DstVT = Op.getValueType();
+ if (DstVT != MVT::f32 && DstVT != MVT::v2f32 && DstVT != MVT::v4f32)
+ return SDValue();
+
+ bool IsBF8 = false;
+ switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
+ case APFloatBase::S_Float8E4M3FN:
+ IsBF8 = false;
+ break;
+ case APFloatBase::S_Float8E5M2:
+ IsBF8 = true;
+ break;
+ default:
+ return SDValue();
+ }
+
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // Defer constant inputs to generic bit-twiddling expansion.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Src))
+ return SDValue();
+
+ // Pack vector inputs as hw instruction is packed.
+ // FIXME: note, packing loses lane-wise poison. Should we do something about
+ // it?
+ auto PackBytes = [&](unsigned NumBytes, unsigned FirstLane) {
+ EVT EltVT = Src.getValueType().getVectorElementType();
+ SDValue PackedI32;
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ SDValue Elt = DAG.getExtractVectorElt(SL, EltVT, Src, FirstLane + I);
+ SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
+ Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
+ DAG.getConstant(0xFF, SL, MVT::i32));
+ if (I != 0)
+ Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
+ DAG.getConstant(I * 8, SL, MVT::i32));
+ PackedI32 =
+ I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
+ }
+ return PackedI32;
+ };
+
+ auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
+ unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
+ : Intrinsic::amdgcn_cvt_pk_f32_fp8;
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI32,
+ DAG.getTargetConstant(WordSel, SL, MVT::i1));
+ };
+
+ if (DstVT == MVT::f32) {
+ unsigned IntrID =
+ IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
+ DAG.getTargetConstant(0, SL, MVT::i32));
+ }
+
+ if (DstVT == MVT::v2f32)
+ return EmitPk(PackBytes(2, 0), 0);
+
+ SDValue PackedI32 = PackBytes(4, 0);
+ SDValue Lo = EmitPk(PackedI32, 0);
+ SDValue Hi = EmitPk(PackedI32, 1);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+}
+
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index e37bd938dc35d..e19e3d580e349 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -103,6 +103,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
new file mode 100644
index 0000000000000..87ea18db6feb8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -0,0 +1,342 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+
+; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
+
+declare float @llvm.convert.from.arbitrary.fp.f32.i8(i8, metadata)
+declare <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8>, metadata)
+declare <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8>, metadata)
+
+; Scalar Float8E5M2
+define float @from_bf8_dynamic(i8 %x) {
+; GFX942-LABEL: from_bf8_dynamic:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_cvt_f32_bf8_sdwa v0, v0 src0_sel:BYTE_0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: from_bf8_dynamic:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: from_bf8_dynamic:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: from_bf8_dynamic:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_bf8_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E5M2")
+ ret float %r
+}
+
+; Scalar Float8E4M3FN
+define float @from_fp8_dynamic(i8 %x) {
+; GFX942-LABEL: from_fp8_dynamic:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_cvt_f32_fp8_sdwa v0, v0 src0_sel:BYTE_0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: from_fp8_dynamic:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: from_fp8_dynamic:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: from_fp8_dynamic:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f32_fp8_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E4M3FN")
+ ret float %r
+}
+
+; v2f32
+define <2 x float> @v2_from_bf8(<2 x i8> %x) {
+; GFX942-LABEL: v2_from_bf8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v2_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v2_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v2_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
+ ret <2 x float> %r
+}
+
+define <2 x float> @v2_from_fp8(<2 x i8> %x) {
+; GFX942-LABEL: v2_from_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v2_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v2_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v2_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
+ ret <2 x float> %r
+}
+
+; v4f32
+define <4 x float> @v4_from_bf8(<4 x i8> %x) {
+; GFX942-LABEL: v4_from_bf8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX942-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v4_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v4_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v4_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
+ ret <4 x float> %r
+}
+
+define <4 x float> @v4_from_fp8(<4 x i8> %x) {
+; GFX942-LABEL: v4_from_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT: s_mov_b32 s0, 0x6050400
+; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX942-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: v4_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX950-NEXT: s_mov_b32 s0, 0x6050400
+; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v4_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v4_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
+ ret <4 x float> %r
+}
+
+; Fall through to the generic expansion in case of constant inputs so they fold..
+define float @const_fp8() {
+; GFX942-LABEL: const_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: const_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: const_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: const_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 56, metadata !"Float8E4M3FN")
+ ret float %r
+}
+
+define <2 x float> @const_v2_fp8() {
+; GFX942-LABEL: const_v2_fp8:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX942-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: const_v2_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX950-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: const_v2_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: const_v2_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> <i8 56, i8 64>, metadata !"Float8E4M3FN")
+ ret <2 x float> %r
+}
>From 38b7e5321bb88091ca7fb1cf63526e5c820572e4 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 26 Apr 2026 23:54:57 +0200
Subject: [PATCH 02/12] add fp16 lowering, address comments and do a bit of
refactoring for next
conversions
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 150 ++++++++++++------
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 84 ++++++++++
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 12 +-
4 files changed, 192 insertions(+), 58 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 37a8a008e0552..9525fe22e853b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -993,6 +993,11 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
Custom);
}
+ if (Subtarget->hasGFX1250Insts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
+ {MVT::f16, MVT::v2f16, MVT::v4f16}, Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -10579,51 +10584,42 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
-SDValue
-SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
- SelectionDAG &DAG) const {
+// Pack vector inputs as hw instruction is packed.
+// FIXME: note, packing loses lane-wise poison. Should we do something about
+// it?
+SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
+ SDValue Src, unsigned NumBytes,
+ unsigned FirstLane) {
+ EVT SrcEltVT = Src.getValueType().getVectorElementType();
+ SDValue PackedI32;
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ SDValue Elt = DAG.getExtractVectorElt(SL, SrcEltVT, Src, FirstLane + I);
+ SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
+ Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
+ DAG.getConstant(0xFF, SL, MVT::i32));
+ if (I != 0)
+ Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
+ DAG.getConstant(I * 8, SL, MVT::i32));
+ PackedI32 =
+ I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
+ }
+ return PackedI32;
+}
+
+SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
EVT DstVT = Op.getValueType();
- if (DstVT != MVT::f32 && DstVT != MVT::v2f32 && DstVT != MVT::v4f32)
- return SDValue();
-
- bool IsBF8 = false;
- switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
- case APFloatBase::S_Float8E4M3FN:
- IsBF8 = false;
- break;
- case APFloatBase::S_Float8E5M2:
- IsBF8 = true;
- break;
- default:
- return SDValue();
- }
-
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- // Defer constant inputs to generic bit-twiddling expansion.
- if (DAG.isConstantIntBuildVectorOrConstantInt(Src))
- return SDValue();
-
- // Pack vector inputs as hw instruction is packed.
- // FIXME: note, packing loses lane-wise poison. Should we do something about
- // it?
- auto PackBytes = [&](unsigned NumBytes, unsigned FirstLane) {
- EVT EltVT = Src.getValueType().getVectorElementType();
- SDValue PackedI32;
- for (unsigned I = 0; I != NumBytes; ++I) {
- SDValue Elt = DAG.getExtractVectorElt(SL, EltVT, Src, FirstLane + I);
- SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
- Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
- DAG.getConstant(0xFF, SL, MVT::i32));
- if (I != 0)
- Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
- DAG.getConstant(I * 8, SL, MVT::i32));
- PackedI32 =
- I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
- }
- return PackedI32;
- };
+ if (DstVT == MVT::f32) {
+ unsigned IntrID =
+ IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
+ DAG.getTargetConstant(0, SL, MVT::i32));
+ }
auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
@@ -10633,22 +10629,76 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
DAG.getTargetConstant(WordSel, SL, MVT::i1));
};
- if (DstVT == MVT::f32) {
+ if (DstVT == MVT::v2f32)
+ return EmitPk(packBytesToI32(DAG, SL, Src, 2, 0), 0);
+
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 4, 0);
+ SDValue Lo = EmitPk(PackedI32, 0);
+ SDValue Hi = EmitPk(PackedI32, 1);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+}
+
+SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasGFX1250Insts() &&
+ "unscaled fp8/bf8 -> f16 requires gfx1250+");
+ EVT DstVT = Op.getValueType();
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // amdgcn_cvt_f16_{fp8,bf8}(i32 src, byte_sel) -> f16.
+ // amdgcn_cvt_pk_f16_{fp8,bf8}(i16 src) -> v2f16.
+ if (!DstVT.isVector()) {
unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
+ IsBF8 ? Intrinsic::amdgcn_cvt_f16_bf8 : Intrinsic::amdgcn_cvt_f16_fp8;
SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f16,
DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
DAG.getTargetConstant(0, SL, MVT::i32));
}
- if (DstVT == MVT::v2f32)
- return EmitPk(PackBytes(2, 0), 0);
+ unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
+ : Intrinsic::amdgcn_cvt_pk_f16_fp8;
+ SDValue IntrConst = DAG.getTargetConstant(IntrID, SL, MVT::i32);
+ auto EmitPk = [&](unsigned FirstLane) {
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 2, FirstLane);
+ SDValue PackedI16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, PackedI32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f16, IntrConst,
+ PackedI16);
+ };
+ if (DstVT.getVectorNumElements() == 2)
+ return EmitPk(0);
+ SDValue Lo = EmitPk(0);
+ SDValue Hi = EmitPk(2);
+ return DAG.getNode(ISD::CONCAT_VECTORS, SL, DstVT, Lo, Hi);
+}
- SDValue PackedI32 = PackBytes(4, 0);
- SDValue Lo = EmitPk(PackedI32, 0);
- SDValue Hi = EmitPk(PackedI32, 1);
- return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ bool IsBF8 = false;
+ switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
+ case APFloatBase::S_Float8E4M3FN:
+ IsBF8 = false;
+ break;
+ case APFloatBase::S_Float8E5M2:
+ IsBF8 = true;
+ break;
+ default:
+ return SDValue();
+ }
+
+ // Defer constant inputs to generic bit-twiddling expansion.
+ if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(0)))
+ return SDValue();
+
+ EVT DstVT = Op.getValueType();
+ EVT EltVT = DstVT.isVector() ? DstVT.getVectorElementType() : DstVT;
+ if (EltVT == MVT::f16)
+ return lowerFromFP8ToF16(Op, IsBF8, DAG);
+ if (DstVT == MVT::f32 || DstVT == MVT::v2f32 || DstVT == MVT::v4f32)
+ return lowerFromFP8ToF32(Op, IsBF8, DAG);
+ return SDValue();
}
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index e19e3d580e349..c0ca80ab23177 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -104,6 +104,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8ToF32(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8ToF16(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
+ static SDValue packBytesToI32(SelectionDAG &DAG, const SDLoc &SL, SDValue Src,
+ unsigned NumBytes, unsigned FirstLane);
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
new file mode 100644
index 0000000000000..752d037cb9691
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+
+; llvm.convert.from.arbitrary.fp from FP8/BF8 producing f16 result types,
+; lowered via the gfx1250+ unscaled v_cvt_{pk_}f16_{fp8,bf8} instructions.
+
+; Scalar Float8E4M3FN
+define half @from_fp8_f16_s(i8 %x) {
+; GFX1250-LABEL: from_fp8_f16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E4M3FN")
+ ret half %r
+}
+
+; Scalar Float8E5M2
+define half @from_bf8_f16_s(i8 %x) {
+; GFX1250-LABEL: from_bf8_f16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E5M2")
+ ret half %r
+}
+
+; Vector Float8E4M3FN
+define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
+ ret <2 x half> %r
+}
+
+; Vector Float8E5M2
+define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
+; GFX1250-LABEL: from_bf8_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
+ ret <2 x half> %r
+}
+
+define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v4f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v1, v3, v2, 0x6050400
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
+ ret <4 x half> %r
+}
+
+; Constant input bypasses HW-specific path.
+define half @from_fp8_f16_const() {
+; GFX1250-LABEL: from_fp8_f16_const:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 56, metadata !"Float8E4M3FN")
+ ret half %r
+}
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index 87ea18db6feb8..2597e607a5a1f 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -1,15 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
-; RUN: llc -global-isel=0 < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
-declare float @llvm.convert.from.arbitrary.fp.f32.i8(i8, metadata)
-declare <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8>, metadata)
-declare <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8>, metadata)
-
; Scalar Float8E5M2
define float @from_bf8_dynamic(i8 %x) {
; GFX942-LABEL: from_bf8_dynamic:
>From 82273bb04779e4d25d51a4b761f14fbb788c8c0c Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 30 Apr 2026 15:08:57 -0500
Subject: [PATCH 03/12] Guard FP8 custom lowering with hasOCPFP8ConversionInsts
---
.../include/llvm/Target/TargetSelectionDAG.td | 5 +
llvm/lib/Target/AMDGPU/AMDGPU.td | 16 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 86 +++---
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 9 +
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 38 ++-
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 30 ++-
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 253 +++++++++---------
7 files changed, 256 insertions(+), 181 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 573342846b4cf..e53ff4d974e2a 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -175,6 +175,9 @@ def SDIsFPClassOp : SDTypeProfile<1, 2, [ // is_fpclass
def SDTIntToFPOp : SDTypeProfile<1, 1, [ // [su]int_to_fp
SDTCisFP<0>, SDTCisInt<1>, SDTCisSameNumEltsAs<0, 1>
]>;
+def SDTConvertFromArbitraryFPOp : SDTypeProfile<1, 2, [
+ SDTCisFP<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisSameNumEltsAs<0, 1>
+]>;
def SDTFPToIntOp : SDTypeProfile<1, 1, [ // fp_to_[su]int
SDTCisInt<0>, SDTCisFP<1>, SDTCisSameNumEltsAs<0, 1>
]>;
@@ -637,6 +640,8 @@ def is_fpclass : SDNode<"ISD::IS_FPCLASS" , SDIsFPClassOp>;
def sint_to_fp : SDNode<"ISD::SINT_TO_FP" , SDTIntToFPOp>;
def uint_to_fp : SDNode<"ISD::UINT_TO_FP" , SDTIntToFPOp>;
+def convert_from_arbitrary_fp : SDNode<"ISD::CONVERT_FROM_ARBITRARY_FP",
+ SDTConvertFromArbitraryFPOp>;
def fp_to_sint : SDNode<"ISD::FP_TO_SINT" , SDTFPToIntOp>;
def fp_to_uint : SDNode<"ISD::FP_TO_UINT" , SDTFPToIntOp>;
def fp_to_sint_sat : SDNode<"ISD::FP_TO_SINT_SAT" , SDTFPToIntSatOp>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 25fc64d178858..7d6b968845731 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -795,6 +795,15 @@ defm FP8ConversionInsts : AMDGPUSubtargetFeature<"fp8-conversion-insts",
"Has fp8 and bf8 conversion instructions"
>;
+// Note: it doesn't include FNUZ dataformat, therefore gfx942 is excluded.
+defm OCPFP8ConversionInsts : AMDGPUSubtargetFeature<"ocp-fp8-conversion-insts",
+ "Has OCP fp8 (E4M3FN, E5M2) conversion instructions"
+>;
+
+defm FP8F16ConversionInsts : AMDGPUSubtargetFeature<"fp8-f16-conversion-insts",
+ "Has fp8 and bf8 conversion to f16 instructions"
+>;
+
defm FP8E5M3Insts : AMDGPUSubtargetFeature<"fp8e5m3-insts",
"Has fp8 e5m3 format support"
>;
@@ -1762,6 +1771,7 @@ def FeatureISAVersion9_5_Common : FeatureSet<
[FeatureAddressableLocalMemorySize163840,
FeatureFP8Insts,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureGFX950Insts,
FeaturePrngInst,
FeatureBF16ConversionInsts,
@@ -1969,6 +1979,7 @@ def FeatureISAVersion11_7_Common : FeatureSet<
FeatureSALUFloatInsts,
FeatureDPPSrc1SGPR,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureDot11Insts,
FeatureWMMA128bInsts,
FeatureSWMMACGfx1200Insts,
@@ -2005,6 +2016,7 @@ def FeatureISAVersion12 : FeatureSet<
FeatureImageInsts,
FeatureExtendedImageInsts,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
FeatureWMMA128bInsts,
FeatureSWMMACGfx1200Insts,
FeatureIEEEMinimumMaximumInsts,
@@ -2060,6 +2072,8 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureAtomicBufferPkAddBF16Inst,
FeatureFlatAtomicFaddF32Inst,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
+ FeatureFP8F16ConversionInsts,
FeatureFP8E5M3Insts,
FeaturePackedTID,
FeatureVcmpxPermlaneHazard,
@@ -2173,6 +2187,8 @@ def FeatureISAVersion13 : FeatureSet<
FeatureAtomicBufferPkAddBF16Inst,
FeatureFlatAtomicFaddF32Inst,
FeatureFP8ConversionInsts,
+ FeatureOCPFP8ConversionInsts,
+ FeatureFP8F16ConversionInsts,
FeaturePackedTID,
FeatureVcmpxPermlaneHazard,
FeatureSALUFloatInsts,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9525fe22e853b..81e84bacde106 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -986,14 +986,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
- if (Subtarget->hasFP8ConversionInsts()) {
+ if (Subtarget->hasOCPFP8ConversionInsts()) {
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
{MVT::f32, MVT::v2f32, MVT::v4f32}, Custom);
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::v2i8, MVT::v4i8},
Custom);
}
- if (Subtarget->hasGFX1250Insts()) {
+ if (Subtarget->hasFP8F16ConversionInsts()) {
setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
{MVT::f16, MVT::v2f16, MVT::v4f16}, Custom);
}
@@ -10584,26 +10584,32 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
-// Pack vector inputs as hw instruction is packed.
-// FIXME: note, packing loses lane-wise poison. Should we do something about
-// it?
+// Pack a vector of i8 lanes into i32 via bitcast.
+// FIXME: packing loses lane-wise poison. Should we do something about it?
SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
SDValue Src, unsigned NumBytes,
unsigned FirstLane) {
- EVT SrcEltVT = Src.getValueType().getVectorElementType();
- SDValue PackedI32;
- for (unsigned I = 0; I != NumBytes; ++I) {
- SDValue Elt = DAG.getExtractVectorElt(SL, SrcEltVT, Src, FirstLane + I);
- SDValue Byte = DAG.getZExtOrTrunc(Elt, SL, MVT::i32);
- Byte = DAG.getNode(ISD::AND, SL, MVT::i32, Byte,
- DAG.getConstant(0xFF, SL, MVT::i32));
- if (I != 0)
- Byte = DAG.getNode(ISD::SHL, SL, MVT::i32, Byte,
- DAG.getConstant(I * 8, SL, MVT::i32));
- PackedI32 =
- I == 0 ? Byte : DAG.getNode(ISD::OR, SL, MVT::i32, PackedI32, Byte);
- }
- return PackedI32;
+ EVT SrcVT = Src.getValueType();
+ assert(SrcVT.isVector() &&
+ SrcVT.getVectorElementType() == MVT::i8 &&
+ "packBytesToI32 expects a v*i8 source");
+ unsigned SrcBytes = SrcVT.getVectorNumElements();
+ assert((NumBytes == 2 || NumBytes == 4) && "expected 2 or 4 bytes");
+
+ if (NumBytes == 4) {
+ assert(FirstLane == 0 && "v4i8 -> i32 takes the full vector");
+ return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Src);
+ }
+
+ // Bitcast a vector to the same-width integer, optionally shift down to the
+ // requested slice, truncate to i16, and zext to i32.
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcBytes * 8);
+ SDValue AsInt = DAG.getNode(ISD::BITCAST, SL, IntVT, Src);
+ if (FirstLane != 0)
+ AsInt = DAG.getNode(ISD::SRL, SL, IntVT, AsInt,
+ DAG.getConstant(FirstLane * 8, SL, IntVT));
+ SDValue I16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, AsInt);
+ return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, I16);
}
SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
@@ -10612,14 +10618,7 @@ SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- if (DstVT == MVT::f32) {
- unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f32_bf8 : Intrinsic::amdgcn_cvt_f32_fp8;
- SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
- DAG.getTargetConstant(0, SL, MVT::i32));
- }
+ assert(DstVT.isVector() && "Scalar f32 should be selected by TableGen");
auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
@@ -10640,22 +10639,14 @@ SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
SelectionDAG &DAG) const {
- assert(Subtarget->hasGFX1250Insts() &&
- "unscaled fp8/bf8 -> f16 requires gfx1250+");
+ assert(Subtarget->hasFP8F16ConversionInsts() &&
+ "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
EVT DstVT = Op.getValueType();
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- // amdgcn_cvt_f16_{fp8,bf8}(i32 src, byte_sel) -> f16.
- // amdgcn_cvt_pk_f16_{fp8,bf8}(i16 src) -> v2f16.
- if (!DstVT.isVector()) {
- unsigned IntrID =
- IsBF8 ? Intrinsic::amdgcn_cvt_f16_bf8 : Intrinsic::amdgcn_cvt_f16_fp8;
- SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f16,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), SrcI32,
- DAG.getTargetConstant(0, SL, MVT::i32));
- }
+ // Scalar f16 is handled by TableGen patterns.
+ assert(DstVT.isVector() && "Scalar f16 should be selected by TableGen");
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
: Intrinsic::amdgcn_cvt_pk_f16_fp8;
@@ -10676,6 +10667,8 @@ SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
SDValue
SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
SelectionDAG &DAG) const {
+ // Only handle OCP FP8 formats (E4M3FN, E5M2). FNUZ formats that are supported
+ // by gfx942 fall through to the generic expansion.
bool IsBF8 = false;
switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
case APFloatBase::S_Float8E4M3FN:
@@ -10693,10 +10686,21 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
return SDValue();
EVT DstVT = Op.getValueType();
- EVT EltVT = DstVT.isVector() ? DstVT.getVectorElementType() : DstVT;
+ if (!DstVT.isVector()) {
+ SDValue Src = Op.getOperand(0);
+ if (Src.getValueType() != MVT::i32) {
+ SDLoc SL(Op);
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, SL, DstVT, SrcI32,
+ Op.getOperand(1));
+ }
+ return Op;
+ }
+
+ EVT EltVT = DstVT.getVectorElementType();
if (EltVT == MVT::f16)
return lowerFromFP8ToF16(Op, IsBF8, DAG);
- if (DstVT == MVT::f32 || DstVT == MVT::v2f32 || DstVT == MVT::v4f32)
+ if (EltVT == MVT::f32)
return lowerFromFP8ToF32(Op, IsBF8, DAG);
return SDValue();
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index ef6d0f22a2f3e..a24624f5e25df 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -911,6 +911,15 @@ def RtzRoundMode : TImmLeaf<i32, [{
return Imm == (int)RoundingMode::TowardZero;
}]>;
+// llvm.convert.from.arbitrary.fp format-selector predicates.
+// Only OCP formats are matched here; FNUZ formats use the generic expansion.
+def fp8_sem : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E4M3FN;
+}]>;
+def bf8_sem : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E5M2;
+}]>;
+
def VOP3PModsNeg : SDNodeXForm<timm, [{
unsigned Mods = SISrcMods::OP_SEL_1;
if (N->getZExtValue())
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 9955d60fb2849..6c675c32f69c2 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -726,6 +726,15 @@ foreach Index = [1, 2, 3] in {
}
} // End OtherPredicates = [HasSDWA]
+// llvm.convert.from.arbitrary.fp i8 -> f32 scalar patterns.
+let OtherPredicates = [NotHasCvtFP8VOP1Bug, HasSDWA] in {
+ // gfx950 has OCP fp8 hardware with SDWA available.
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_e32 $src)>;
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 bf8_sem))),
+ (V_CVT_F32_BF8_e32 $src)>;
+}
+
} // End SubtargetPredicate = HasFP8ConversionInsts
class Cvt_PK_F32_F8_Pat<SDPatternOperator node, int index,
@@ -811,6 +820,17 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
}
let SubtargetPredicate = isGFX11Plus in
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
+
+ // llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns.
+ let SubtargetPredicate = isGFX11PlusNot12_50 in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
+ let SubtargetPredicate = isGFX125xOnly in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
+ let SubtargetPredicate = isGFX11Plus in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
}
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
@@ -837,7 +857,7 @@ def VOPProfile_CVT_PK_F16_F8_true16 : VOP3_Profile_True16<VOPProfile_CVT_PK_F16_
def VOPProfile_CVT_PK_F16_F8_fake16 : VOP3_Profile_Fake16<VOPProfile_CVT_PK_F16_F8>;
}
-let SubtargetPredicate = isGFX1250Plus in {
+let SubtargetPredicate = HasFP8F16ConversionInsts in {
let mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
defm V_CVT_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_fp8",
V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
@@ -860,6 +880,22 @@ let SubtargetPredicate = isGFX1250Plus in {
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
}
+ // llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+ let True16Predicate = UseRealTrue16Insts in {
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F16_FP8_t16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F16_BF8_t16_e64 0, $src0, 0)>;
+ }
+ let True16Predicate = UseFakeTrue16Insts in {
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ (V_CVT_F16_FP8_fake16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ (V_CVT_F16_BF8_fake16_e64 0, $src0, 0)>;
+ }
+} // End SubtargetPredicate = HasFP8F16ConversionInsts
+
+let SubtargetPredicate = isGFX1250Plus in {
defm V_SAT_PK4_I4_I8 : VOP1Inst_t16<"v_sat_pk4_i4_i8", VOP1_I16_I32, int_amdgcn_sat_pk4_i4_i8>;
defm V_SAT_PK4_U4_U8 : VOP1Inst_t16<"v_sat_pk4_u4_u8", VOP1_I16_I32, int_amdgcn_sat_pk4_u4_u8>;
} // End SubtargetPredicate = isGFX1250Plus
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index 752d037cb9691..7d63187038f61 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -34,8 +34,9 @@ define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
@@ -48,21 +49,38 @@ define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
ret <2 x half> %r
}
+; v3f16
+define <3 x half> @from_fp8_v3f16(<3 x i8> %x) {
+; GFX1250-LABEL: from_fp8_v3f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
+ ret <3 x half> %r
+}
+
define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
; GFX1250-LABEL: from_fp8_v4f16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_perm_b32 v1, v3, v2, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index 2597e607a5a1f..ba129f08406fb 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -1,19 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 | FileCheck -check-prefix=GFX942 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 | FileCheck -check-prefix=GFX950 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1170 | FileCheck -check-prefix=GFX1170 %s
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
-; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets.
+; llvm.convert.from.arbitrary.fp on FP8ConversionInsts targets with OCP FP8.
; Scalar Float8E5M2
define float @from_bf8_dynamic(i8 %x) {
-; GFX942-LABEL: from_bf8_dynamic:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_cvt_f32_bf8_sdwa v0, v0 src0_sel:BYTE_0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: from_bf8_dynamic:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -38,12 +31,6 @@ define float @from_bf8_dynamic(i8 %x) {
; Scalar Float8E4M3FN
define float @from_fp8_dynamic(i8 %x) {
-; GFX942-LABEL: from_fp8_dynamic:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_cvt_f32_fp8_sdwa v0, v0 src0_sel:BYTE_0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: from_fp8_dynamic:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -68,30 +55,19 @@ define float @from_fp8_dynamic(i8 %x) {
; v2f32
define <2 x float> @v2_from_bf8(<2 x i8> %x) {
-; GFX942-LABEL: v2_from_bf8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v2_from_bf8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v2_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
@@ -99,9 +75,8 @@ define <2 x float> @v2_from_bf8(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
@@ -109,30 +84,19 @@ define <2 x float> @v2_from_bf8(<2 x i8> %x) {
}
define <2 x float> @v2_from_fp8(<2 x i8> %x) {
-; GFX942-LABEL: v2_from_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v2_from_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v2_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
@@ -140,9 +104,8 @@ define <2 x float> @v2_from_fp8(<2 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
@@ -151,30 +114,13 @@ define <2 x float> @v2_from_fp8(<2 x i8> %x) {
; v4f32
define <4 x float> @v4_from_bf8(<4 x i8> %x) {
-; GFX942-LABEL: v4_from_bf8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
-; GFX942-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX942-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v4_from_bf8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
+; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
; GFX950-NEXT: s_setpc_b64 s[30:31]
@@ -182,14 +128,10 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX1170-LABEL: v4_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1170-NEXT: s_setpc_b64 s[30:31]
@@ -198,14 +140,10 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -214,30 +152,13 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
}
define <4 x float> @v4_from_fp8(<4 x i8> %x) {
-; GFX942-LABEL: v4_from_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX942-NEXT: s_mov_b32 s0, 0x6050400
-; GFX942-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX942-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX942-NEXT: v_or3_b32 v2, v0, v2, v1
-; GFX942-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX942-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: v4_from_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX950-NEXT: s_mov_b32 s0, 0x6050400
-; GFX950-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX950-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 24, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX950-NEXT: v_or3_b32 v2, v0, v2, v1
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
+; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
; GFX950-NEXT: s_setpc_b64 s[30:31]
@@ -245,14 +166,10 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX1170-LABEL: v4_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1170-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1170-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1170-NEXT: s_setpc_b64 s[30:31]
@@ -261,14 +178,10 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX1250-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x6050400
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_or3_b32 v2, v0, v1, v3
+; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -276,14 +189,95 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
ret <4 x float> %r
}
+; v3f32
+define <3 x float> @v3_from_bf8(<3 x i8> %x) {
+; GFX950-LABEL: v3_from_bf8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v3_from_bf8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v3_from_bf8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E5M2")
+ ret <3 x float> %r
+}
+
+define <3 x float> @v3_from_fp8(<3 x i8> %x) {
+; GFX950-LABEL: v3_from_fp8:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
+; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
+; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1170-LABEL: v3_from_fp8:
+; GFX1170: ; %bb.0:
+; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v3_from_fp8:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
+ ret <3 x float> %r
+}
+
; Fall through to the generic expansion in case of constant inputs so they fold..
define float @const_fp8() {
-; GFX942-LABEL: const_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: const_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -307,13 +301,6 @@ define float @const_fp8() {
}
define <2 x float> @const_v2_fp8() {
-; GFX942-LABEL: const_v2_fp8:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
; GFX950-LABEL: const_v2_fp8:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
>From fce9b15f84109cfd973e64d158dd902044d4f38e Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 1 May 2026 01:03:47 +0200
Subject: [PATCH 04/12] format
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 91b3c112f8348..6fb8aafa328f8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10590,8 +10590,7 @@ SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
SDValue Src, unsigned NumBytes,
unsigned FirstLane) {
EVT SrcVT = Src.getValueType();
- assert(SrcVT.isVector() &&
- SrcVT.getVectorElementType() == MVT::i8 &&
+ assert(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i8 &&
"packBytesToI32 expects a v*i8 source");
unsigned SrcBytes = SrcVT.getVectorNumElements();
assert((NumBytes == 2 || NumBytes == 4) && "expected 2 or 4 bytes");
>From c5e585fa2b8bb400edfe7be1ea25b58f67bd7e7a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 2 May 2026 12:44:34 +0200
Subject: [PATCH 05/12] Apply comments, move to TargetSelectionDAG.td
---
.../include/llvm/Target/TargetSelectionDAG.td | 9 ++++
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 9 ----
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 46 ++++++++++---------
.../GlobalISelEmitter/GlobalISelEmitter.td | 10 +++-
4 files changed, 43 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index e53ff4d974e2a..00c1b29227a61 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -1264,6 +1264,15 @@ class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
SDNode ImmNode = timm> : ImmLeaf<vt, pred, xform, ImmNode>;
+// llvm.convert.from.arbitrary.fp format-selector predicates.
+// Only OCP formats are matched here; FNUZ formats use the generic expansion.
+def Float8E4M3FN : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E4M3FN;
+}]>;
+def Float8E5M2 : TImmLeaf<i32, [{
+ return Imm == APFloatBase::S_Float8E5M2;
+}]>;
+
// An ImmLeaf where the `Imm` in the C++ predicate is an `APInt`.
//
// This is useful when you need to zero-extend the immediate instead of
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 8b01adb7ef50a..7c3aff9c9f47f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -911,15 +911,6 @@ def RtzRoundMode : TImmLeaf<i32, [{
return Imm == (int)RoundingMode::TowardZero;
}]>;
-// llvm.convert.from.arbitrary.fp format-selector predicates.
-// Only OCP formats are matched here; FNUZ formats use the generic expansion.
-def fp8_sem : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E4M3FN;
-}]>;
-def bf8_sem : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E5M2;
-}]>;
-
def VOP3PModsNeg : SDNodeXForm<timm, [{
unsigned Mods = SISrcMods::OP_SEL_1;
if (N->getZExtValue())
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 6c675c32f69c2..5725b47a3598e 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -727,11 +727,11 @@ foreach Index = [1, 2, 3] in {
} // End OtherPredicates = [HasSDWA]
// llvm.convert.from.arbitrary.fp i8 -> f32 scalar patterns.
-let OtherPredicates = [NotHasCvtFP8VOP1Bug, HasSDWA] in {
- // gfx950 has OCP fp8 hardware with SDWA available.
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 fp8_sem))),
+// gfx950 has SDWA and uses the e32 form.
+let OtherPredicates = [HasOCPFP8ConversionInsts, HasSDWA] in {
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
(V_CVT_F32_FP8_e32 $src)>;
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, (i32 bf8_sem))),
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
(V_CVT_F32_BF8_e32 $src)>;
}
@@ -820,19 +820,20 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
}
let SubtargetPredicate = isGFX11Plus in
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
-
- // llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns.
- let SubtargetPredicate = isGFX11PlusNot12_50 in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
- (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
- let SubtargetPredicate = isGFX125xOnly in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
- (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
- let SubtargetPredicate = isGFX11Plus in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
- (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
}
+// llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns for GFX11+.
+// GFX11+ doesn't have SDWA, so must use OP_SEL form.
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus, isNotGFX1250Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
+ (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX1250Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
+ (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus] in
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
+ (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
+
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
VOP1_Pseudo inst_e32, VOP3_Pseudo inst_e64> : GCNPat<
(v2f32 (node i32:$src, index)),
@@ -880,20 +881,23 @@ let SubtargetPredicate = HasFP8F16ConversionInsts in {
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
}
- // llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+} // End SubtargetPredicate = HasFP8F16ConversionInsts
+
+// llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
+let OtherPredicates = [HasOCPFP8ConversionInsts, HasFP8F16ConversionInsts] in {
let True16Predicate = UseRealTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
(V_CVT_F16_FP8_t16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
(V_CVT_F16_BF8_t16_e64 0, $src0, 0)>;
}
let True16Predicate = UseFakeTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 fp8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
(V_CVT_F16_FP8_fake16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, (i32 bf8_sem))),
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
(V_CVT_F16_BF8_fake16_e64 0, $src0, 0)>;
}
-} // End SubtargetPredicate = HasFP8F16ConversionInsts
+}
let SubtargetPredicate = isGFX1250Plus in {
defm V_SAT_PK4_I4_I8 : VOP1Inst_t16<"v_sat_pk4_i4_i8", VOP1_I16_I32, int_amdgcn_sat_pk4_i4_i8>;
diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
index d09034fd53eca..6191170d0f031 100644
--- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
@@ -201,12 +201,20 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
// CHECK-LABEL: // PatFrag predicates.
// CHECK-NEXT: enum {
-// CHECK-NEXT: GICXXPred_I64_Predicate_cimm8 = GICXXPred_Invalid + 1,
+// CHECK-NEXT: GICXXPred_I64_Predicate_Float8E4M3FN = GICXXPred_Invalid + 1,
+// CHECK-NEXT: GICXXPred_I64_Predicate_Float8E5M2,
+// CHECK-NEXT: GICXXPred_I64_Predicate_cimm8,
// CHECK-NEXT: GICXXPred_I64_Predicate_simm8,
// CHECK-NEXT: };
// CHECK-NEXT: bool MyTargetInstructionSelector::testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const {
// CHECK-NEXT: switch (PredicateID) {
+// CHECK-NEXT: case GICXXPred_I64_Predicate_Float8E4M3FN: {
+// CHECK-NEXT: return Imm == APFloatBase::S_Float8E4M3FN;
+// CHECK-NEXT: }
+// CHECK-NEXT: case GICXXPred_I64_Predicate_Float8E5M2: {
+// CHECK-NEXT: return Imm == APFloatBase::S_Float8E5M2;
+// CHECK-NEXT: }
// CHECK-NEXT: case GICXXPred_I64_Predicate_cimm8: {
// CHECK-NEXT: return isInt<8>(Imm);
// CHECK-NEXT: }
>From 3e8a537d26fa4880d7801ceed986c8ef2b6270e6 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 5 May 2026 11:13:48 -0500
Subject: [PATCH 06/12] address review comments and add few more tests
---
.../include/llvm/Target/TargetSelectionDAG.td | 8 +-
.../SelectionDAG/LegalizeVectorOps.cpp | 1 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 6 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 99 ++++--------
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 148 +++++++++++-------
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 116 ++++++--------
7 files changed, 178 insertions(+), 204 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 00c1b29227a61..268e627a7b2ab 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -1266,12 +1266,8 @@ class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
// llvm.convert.from.arbitrary.fp format-selector predicates.
// Only OCP formats are matched here; FNUZ formats use the generic expansion.
-def Float8E4M3FN : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E4M3FN;
-}]>;
-def Float8E5M2 : TImmLeaf<i32, [{
- return Imm == APFloatBase::S_Float8E5M2;
-}]>;
+def Float8E4M3FN : TImmLeaf<i32, [{ return Imm == APFloatBase::S_Float8E4M3FN; }]>;
+def Float8E5M2 : TImmLeaf<i32, [{ return Imm == APFloatBase::S_Float8E5M2; }]>;
// An ImmLeaf where the `Imm` in the C++ predicate is an `APInt`.
//
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 1aa21fc636a9d..cef12768cb183 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1406,6 +1406,7 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::FROUNDEVEN:
case ISD::FTRUNC:
case ISD::FSQRT:
+ case ISD::CONVERT_FROM_ARBITRARY_FP:
if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
Results.push_back(Expanded);
return;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index f37ea127dd357..5c933958adc4a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13196,6 +13196,12 @@ SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
SmallVector<SDValue, 4> LoOps, HiOps;
for (const SDValue &V : Node->op_values()) {
+ if (!V.getValueType().isVector()) {
+ // Scalar operands pass through to both halves unchanged.
+ LoOps.push_back(V);
+ HiOps.push_back(V);
+ continue;
+ }
auto [Lo, Hi] = DAG.SplitVector(V, DL, LoVT, HiVT);
LoOps.push_back(Lo);
HiOps.push_back(Hi);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 6fb8aafa328f8..0425420e2f0d8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -987,15 +987,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
}
if (Subtarget->hasOCPFP8ConversionInsts()) {
- setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
- {MVT::f32, MVT::v2f32, MVT::v4f32}, Custom);
- setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::v2i8, MVT::v4i8},
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f32, MVT::v2f32},
Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, MVT::v2i8, Custom);
}
if (Subtarget->hasFP8F16ConversionInsts()) {
- setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP,
- {MVT::f16, MVT::v2f16, MVT::v4f16}, Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f16, MVT::v2f16},
+ Custom);
}
if (Subtarget->hasCvtPkF16F32Inst()) {
@@ -10587,80 +10586,40 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
// Pack a vector of i8 lanes into i32 via bitcast.
// FIXME: packing loses lane-wise poison. Should we do something about it?
SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
- SDValue Src, unsigned NumBytes,
- unsigned FirstLane) {
- EVT SrcVT = Src.getValueType();
- assert(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i8 &&
- "packBytesToI32 expects a v*i8 source");
- unsigned SrcBytes = SrcVT.getVectorNumElements();
- assert((NumBytes == 2 || NumBytes == 4) && "expected 2 or 4 bytes");
-
- if (NumBytes == 4) {
- assert(FirstLane == 0 && "v4i8 -> i32 takes the full vector");
- return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Src);
- }
-
- // Bitcast a vector to the same-width integer, optionally shift down to the
- // requested slice, truncate to i16, and zext to i32.
- EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcBytes * 8);
- SDValue AsInt = DAG.getNode(ISD::BITCAST, SL, IntVT, Src);
- if (FirstLane != 0)
- AsInt = DAG.getNode(ISD::SRL, SL, IntVT, AsInt,
- DAG.getConstant(FirstLane * 8, SL, IntVT));
- SDValue I16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, AsInt);
- return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, I16);
+ SDValue Src) {
+ assert(Src.getValueType() == MVT::v2i8 &&
+ "packBytesToI32 expects a v2i8 source");
+ // Widen v2i8 to v4i8 with the high half undef, then bitcast to i32. This
+ // selects to v_perm_b32, packing two i8 values into low 16 bits of an i32.
+ SDValue Wide = DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4i8, Src,
+ DAG.getUNDEF(MVT::v2i8));
+ return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Wide);
}
SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SelectionDAG &DAG) const {
- EVT DstVT = Op.getValueType();
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
-
- assert(DstVT.isVector() && "Scalar f32 should be selected by TableGen");
-
- auto EmitPk = [&](SDValue PackedI32, unsigned WordSel) {
- unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
- : Intrinsic::amdgcn_cvt_pk_f32_fp8;
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f32,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI32,
- DAG.getTargetConstant(WordSel, SL, MVT::i1));
- };
-
- if (DstVT == MVT::v2f32)
- return EmitPk(packBytesToI32(DAG, SL, Src, 2, 0), 0);
-
- SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 4, 0);
- SDValue Lo = EmitPk(PackedI32, 0);
- SDValue Hi = EmitPk(PackedI32, 1);
- return DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4f32, Lo, Hi);
+ unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
+ : Intrinsic::amdgcn_cvt_pk_f32_fp8;
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f32,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI32,
+ DAG.getTargetConstant(0, SL, MVT::i1));
}
SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
SelectionDAG &DAG) const {
assert(Subtarget->hasFP8F16ConversionInsts() &&
"fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
- EVT DstVT = Op.getValueType();
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
-
- // Scalar f16 is handled by TableGen patterns.
- assert(DstVT.isVector() && "Scalar f16 should be selected by TableGen");
-
unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
: Intrinsic::amdgcn_cvt_pk_f16_fp8;
- SDValue IntrConst = DAG.getTargetConstant(IntrID, SL, MVT::i32);
- auto EmitPk = [&](unsigned FirstLane) {
- SDValue PackedI32 = packBytesToI32(DAG, SL, Src, 2, FirstLane);
- SDValue PackedI16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, PackedI32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f16, IntrConst,
- PackedI16);
- };
- if (DstVT.getVectorNumElements() == 2)
- return EmitPk(0);
- SDValue Lo = EmitPk(0);
- SDValue Hi = EmitPk(2);
- return DAG.getNode(ISD::CONCAT_VECTORS, SL, DstVT, Lo, Hi);
+ SDValue PackedI32 = packBytesToI32(DAG, SL, Src);
+ SDValue PackedI16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, PackedI32);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f16,
+ DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI16);
}
SDValue
@@ -10668,17 +10627,13 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
SelectionDAG &DAG) const {
// Only handle OCP FP8 formats (E4M3FN, E5M2). FNUZ formats that are supported
// by gfx942 fall through to the generic expansion.
- bool IsBF8 = false;
- switch (static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1))) {
- case APFloatBase::S_Float8E4M3FN:
- IsBF8 = false;
- break;
- case APFloatBase::S_Float8E5M2:
- IsBF8 = true;
- break;
- default:
+ APFloatBase::Semantics FPSemantic =
+ static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
+ if (FPSemantic != APFloatBase::S_Float8E4M3FN &&
+ FPSemantic != APFloatBase::S_Float8E5M2) {
return SDValue();
}
+ const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
// Defer constant inputs to generic bit-twiddling expansion.
if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(0)))
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c0ca80ab23177..8c1d792f429a8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -106,8 +106,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFromFP8ToF32(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
SDValue lowerFromFP8ToF16(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
- static SDValue packBytesToI32(SelectionDAG &DAG, const SDLoc &SL, SDValue Src,
- unsigned NumBytes, unsigned FirstLane);
+ static SDValue packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
+ SDValue Src);
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index 7d63187038f61..5ccc2100b153b 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -1,102 +1,134 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1251 | FileCheck %s
; llvm.convert.from.arbitrary.fp from FP8/BF8 producing f16 result types,
; lowered via the gfx1250+ unscaled v_cvt_{pk_}f16_{fp8,bf8} instructions.
; Scalar Float8E4M3FN
define half @from_fp8_f16_s(i8 %x) {
-; GFX1250-LABEL: from_fp8_f16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_fp8_f16_s:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E4M3FN")
ret half %r
}
; Scalar Float8E5M2
define half @from_bf8_f16_s(i8 %x) {
-; GFX1250-LABEL: from_bf8_f16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_bf8_f16_s:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E5M2")
ret half %r
}
; Vector Float8E4M3FN
define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
-; GFX1250-LABEL: from_fp8_v2f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
-; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_fp8_v2f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
ret <2 x half> %r
}
; Vector Float8E5M2
define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
-; GFX1250-LABEL: from_bf8_v2f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
-; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_bf8_v2f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
ret <2 x half> %r
}
-; v3f16
+; v3f16 Float8E4M3FN
define <3 x half> @from_fp8_v3f16(<3 x i8> %x) {
-; GFX1250-LABEL: from_fp8_v3f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_fp8_v3f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; CHECK-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
ret <3 x half> %r
}
+; v3f16 Float8E5M2
+define <3 x half> @from_bf8_v3f16(<3 x i8> %x) {
+; CHECK-LABEL: from_bf8_v3f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; CHECK-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E5M2")
+ ret <3 x half> %r
+}
+
+; v4f16 Float8E4M3FN
define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
-; GFX1250-LABEL: from_fp8_v4f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; GFX1250-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_fp8_v4f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; CHECK-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
ret <4 x half> %r
}
+; v4f16 Float8E5M2
+define <4 x half> @from_bf8_v4f16(<4 x i8> %x) {
+; CHECK-LABEL: from_bf8_v4f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; CHECK-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; CHECK-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
+ %r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
+ ret <4 x half> %r
+}
+
; Constant input bypasses HW-specific path.
define half @from_fp8_f16_const() {
-; GFX1250-LABEL: from_fp8_f16_const:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, 0x3c00
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; CHECK-LABEL: from_fp8_f16_const:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 56, metadata !"Float8E4M3FN")
ret half %r
}
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index ba129f08406fb..200f5a43698b5 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -119,21 +119,19 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
-; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: v_perm_b32 v2, v2, v3, s0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v4_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v4_from_bf8:
@@ -141,11 +139,10 @@ define <4 x float> @v4_from_bf8(<4 x i8> %x) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
ret <4 x float> %r
@@ -157,21 +154,19 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0
-; GFX950-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: v_perm_b32 v2, v2, v3, s0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v4_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1170-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v4_from_fp8:
@@ -179,11 +174,10 @@ define <4 x float> @v4_from_fp8(<4 x i8> %x) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
ret <4 x float> %r
@@ -196,38 +190,33 @@ define <3 x float> @v3_from_bf8(<3 x i8> %x) {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX950-NEXT: v_cvt_pk_f32_bf8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_mov_b32 s0, 0xffff
+; GFX950-NEXT: v_and_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX950-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v3_from_bf8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX1170-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1170-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v3_from_bf8:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v2
-; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], v0
+; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[2:3], v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E5M2")
ret <3 x float> %r
@@ -239,38 +228,33 @@ define <3 x float> @v3_from_fp8(<3 x i8> %x) {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX950-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX950-NEXT: v_cvt_pk_f32_fp8_sdwa v[2:3], v2 src0_sel:WORD_1
+; GFX950-NEXT: s_mov_b32 s0, 0xffff
+; GFX950-NEXT: v_and_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX950-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: v3_from_fp8:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1170-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX1170-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX1170-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1170-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX1170-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1170-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v3_from_fp8:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX1250-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX1250-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v2
-; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[2:3], v2 op_sel:[1,0]
+; GFX1250-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX1250-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0
+; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[2:3], v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
ret <3 x float> %r
>From 9c866c5725e4d892b4a9e8bab19248b43a691dfb Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 12 May 2026 11:15:48 -0500
Subject: [PATCH 07/12] fix build after merge with upstream; a bit on nfc
changes
---
.../SelectionDAG/LegalizeVectorOps.cpp | 5 +++-
llvm/lib/Target/AMDGPU/AMDGPU.td | 3 +-
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 28 +++++++++----------
3 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 09ab105bb8a2d..549e20db97a0a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1406,13 +1406,16 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::FROUNDEVEN:
case ISD::FTRUNC:
case ISD::FSQRT:
- case ISD::CONVERT_FROM_ARBITRARY_FP:
if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
Results.push_back(Expanded);
return;
}
break;
case ISD::CONVERT_FROM_ARBITRARY_FP:
+ if (SDValue Expanded = TLI.expandVectorNaryOpBySplitting(Node, DAG)) {
+ Results.push_back(Expanded);
+ return;
+ }
if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
Results.push_back(Expanded);
else
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 75dedcffacbd5..d0244f5eb630f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -807,7 +807,8 @@ defm FP8ConversionInsts : AMDGPUSubtargetFeature<"fp8-conversion-insts",
"Has fp8 and bf8 conversion instructions"
>;
-// Note: it doesn't include FNUZ dataformat, therefore gfx942 is excluded.
+// OCP fp8 semantics (E4M3FN, E5M2). Distinct from the FNUZ formats used by
+// gfx942.
defm OCPFP8ConversionInsts : AMDGPUSubtargetFeature<"ocp-fp8-conversion-insts",
"Has OCP fp8 (E4M3FN, E5M2) conversion instructions"
>;
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index a90b7359377f8..fc1fcc0e26632 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -832,14 +832,14 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
// llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns for GFX11+.
// GFX11+ doesn't have SDWA, so must use OP_SEL form.
let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus, isNotGFX1250Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
- (V_CVT_F32_FP8_OP_SEL_e64 $src0, 0)>;
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
+ (V_CVT_F32_FP8_OP_SEL_e64 $src, 0)>;
let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX1250Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
- (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.NONE, 0)>;
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
+ (V_CVT_F32_FP8_gfx1250_e64 $src, DSTCLAMP.NONE, 0)>;
let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
- (V_CVT_F32_BF8_OP_SEL_e64 $src0, 0)>;
+ def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
+ (V_CVT_F32_BF8_OP_SEL_e64 $src, 0)>;
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
VOP1_Pseudo inst_e32, VOP3_Pseudo inst_e64> : GCNPat<
@@ -893,16 +893,16 @@ let SubtargetPredicate = HasFP8F16ConversionInsts in {
// llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
let OtherPredicates = [HasOCPFP8ConversionInsts, HasFP8F16ConversionInsts] in {
let True16Predicate = UseRealTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
- (V_CVT_F16_FP8_t16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
- (V_CVT_F16_BF8_t16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
+ (V_CVT_F16_FP8_t16_e64 0, $src, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
+ (V_CVT_F16_BF8_t16_e64 0, $src, 0)>;
}
let True16Predicate = UseFakeTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E4M3FN)),
- (V_CVT_F16_FP8_fake16_e64 0, $src0, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src0, Float8E5M2)),
- (V_CVT_F16_BF8_fake16_e64 0, $src0, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
+ (V_CVT_F16_FP8_fake16_e64 0, $src, 0)>;
+ def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
+ (V_CVT_F16_BF8_fake16_e64 0, $src, 0)>;
}
}
>From 93b56dea16ee1dceb0a24bd067fbd69d77293872 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 13 May 2026 15:35:47 +0200
Subject: [PATCH 08/12] Address review
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 ++-
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 53 ++++++----------------
2 files changed, 17 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 740ec1b8659e3..86577a62f12ca 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10689,10 +10689,10 @@ SDValue SITargetLowering::packBytesToI32(SelectionDAG &DAG, const SDLoc &SL,
SDValue Src) {
assert(Src.getValueType() == MVT::v2i8 &&
"packBytesToI32 expects a v2i8 source");
- // Widen v2i8 to v4i8 with the high half undef, then bitcast to i32. This
+ // Widen v2i8 to v4i8 with the high half poison, then bitcast to i32. This
// selects to v_perm_b32, packing two i8 values into low 16 bits of an i32.
SDValue Wide = DAG.getNode(ISD::CONCAT_VECTORS, SL, MVT::v4i8, Src,
- DAG.getUNDEF(MVT::v2i8));
+ DAG.getPOISON(MVT::v2i8));
return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Wide);
}
@@ -10730,9 +10730,8 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
APFloatBase::Semantics FPSemantic =
static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
if (FPSemantic != APFloatBase::S_Float8E4M3FN &&
- FPSemantic != APFloatBase::S_Float8E5M2) {
+ FPSemantic != APFloatBase::S_Float8E5M2)
return SDValue();
- }
const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
// Defer constant inputs to generic bit-twiddling expansion.
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index fc1fcc0e26632..3a11678521d53 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -711,6 +711,11 @@ class Cvt_F32_F8_Pat<SDPatternOperator node, int index,
(inst_sdwa 0, $src, 0, 0, index)
>;
+def cvt_from_fp8 : PatFrag<(ops node:$src),
+ (convert_from_arbitrary_fp node:$src, Float8E4M3FN)>;
+def cvt_from_bf8 : PatFrag<(ops node:$src),
+ (convert_from_arbitrary_fp node:$src, Float8E5M2)>;
+
let SubtargetPredicate = HasFP8ConversionInsts in {
let OtherPredicates = [HasCvtFP8VOP1Bug] in {
def : GCNPat<(f32 (int_amdgcn_cvt_f32_fp8 i32:$src, 0)),
@@ -733,13 +738,9 @@ foreach Index = [1, 2, 3] in {
}
} // End OtherPredicates = [HasSDWA]
-// llvm.convert.from.arbitrary.fp i8 -> f32 scalar patterns.
-// gfx950 has SDWA and uses the e32 form.
let OtherPredicates = [HasOCPFP8ConversionInsts, HasSDWA] in {
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
- (V_CVT_F32_FP8_e32 $src)>;
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
- (V_CVT_F32_BF8_e32 $src)>;
+ def : GCNPat<(f32 (cvt_from_fp8 i32:$src)), (V_CVT_F32_FP8_e32 $src)>;
+ def : GCNPat<(f32 (cvt_from_bf8 i32:$src)), (V_CVT_F32_BF8_e32 $src)>;
}
} // End SubtargetPredicate = HasFP8ConversionInsts
@@ -794,11 +795,11 @@ def V_CVT_F16_F8_Fake16_Profile : VOP3_Profile_Fake16<V_CVT_F16_F8_Profile>;
let SubtargetPredicate = isGFX11Plus, OtherPredicates = [HasFP8ConversionInsts],
mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
let SubtargetPredicate = isGFX11PlusNot12_50 in
- defm V_CVT_F32_FP8_OP_SEL : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
+ defm V_CVT_F32_FP8_OP_SEL : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>, cvt_from_fp8>;
let SubtargetPredicate = isGFX125xOnly in
- defm V_CVT_F32_FP8_gfx1250 : VOP1Inst<"v_cvt_f32_fp8_gfx1250", VOPProfile_Base_CVT_F_F8_ByteSel<f32, 1>>;
+ defm V_CVT_F32_FP8_gfx1250 : VOP1Inst<"v_cvt_f32_fp8_gfx1250", VOPProfile_Base_CVT_F_F8_ByteSel<f32, 1>, cvt_from_fp8>;
- defm V_CVT_F32_BF8_OP_SEL : VOP1Inst<"v_cvt_f32_bf8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
+ defm V_CVT_F32_BF8_OP_SEL : VOP1Inst<"v_cvt_f32_bf8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>, cvt_from_bf8>;
let True16Predicate = UseFakeTrue16Insts in {
defm V_CVT_PK_F32_FP8_fake16 : VOP1Inst<"v_cvt_pk_f32_fp8_fake16", VOPProfile_Base_CVT_PK_F32_F8_fake16>;
@@ -829,18 +830,6 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
}
-// llvm.convert.from.arbitrary.fp i8 -> f32 OCP scalar patterns for GFX11+.
-// GFX11+ doesn't have SDWA, so must use OP_SEL form.
-let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus, isNotGFX1250Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
- (V_CVT_F32_FP8_OP_SEL_e64 $src, 0)>;
-let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX1250Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
- (V_CVT_F32_FP8_gfx1250_e64 $src, DSTCLAMP.NONE, 0)>;
-let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX11Plus] in
- def : GCNPat<(f32 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
- (V_CVT_F32_BF8_OP_SEL_e64 $src, 0)>;
-
class Cvt_PK_F32_F8_Pat_OpSel<SDPatternOperator node, int index,
VOP1_Pseudo inst_e32, VOP3_Pseudo inst_e64> : GCNPat<
(v2f32 (node i32:$src, index)),
@@ -868,9 +857,11 @@ def VOPProfile_CVT_PK_F16_F8_fake16 : VOP3_Profile_Fake16<VOPProfile_CVT_PK_F16_
let SubtargetPredicate = HasFP8F16ConversionInsts in {
let mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
defm V_CVT_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_fp8",
- V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
+ V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile,
+ cvt_from_fp8>;
defm V_CVT_F16_BF8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_bf8",
- V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
+ V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile,
+ cvt_from_bf8>;
defm V_CVT_PK_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_fp8",
VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
int_amdgcn_cvt_pk_f16_fp8>;
@@ -890,22 +881,6 @@ let SubtargetPredicate = HasFP8F16ConversionInsts in {
} // End SubtargetPredicate = HasFP8F16ConversionInsts
-// llvm.convert.from.arbitrary.fp i8 -> f16 OCP scalar patterns.
-let OtherPredicates = [HasOCPFP8ConversionInsts, HasFP8F16ConversionInsts] in {
- let True16Predicate = UseRealTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
- (V_CVT_F16_FP8_t16_e64 0, $src, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
- (V_CVT_F16_BF8_t16_e64 0, $src, 0)>;
- }
- let True16Predicate = UseFakeTrue16Insts in {
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E4M3FN)),
- (V_CVT_F16_FP8_fake16_e64 0, $src, 0)>;
- def : GCNPat<(f16 (convert_from_arbitrary_fp i32:$src, Float8E5M2)),
- (V_CVT_F16_BF8_fake16_e64 0, $src, 0)>;
- }
-}
-
let SubtargetPredicate = isGFX1250Plus in {
defm V_SAT_PK4_I4_I8 : VOP1Inst_t16<"v_sat_pk4_i4_i8", VOP1_I16_I32, int_amdgcn_sat_pk4_i4_i8>;
defm V_SAT_PK4_U4_U8 : VOP1Inst_t16<"v_sat_pk4_u4_u8", VOP1_I16_I32, int_amdgcn_sat_pk4_u4_u8>;
>From 3d5f3fd60f59961ffe95eac562431554ec81e36e Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 13 May 2026 17:57:12 +0200
Subject: [PATCH 09/12] remove constant folding
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 --
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 12 -----
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 47 -------------------
3 files changed, 63 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 86577a62f12ca..d90843c820775 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10734,10 +10734,6 @@ SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
return SDValue();
const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
- // Defer constant inputs to generic bit-twiddling expansion.
- if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(0)))
- return SDValue();
-
EVT DstVT = Op.getValueType();
if (!DstVT.isVector()) {
SDValue Src = Op.getOperand(0);
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index 5ccc2100b153b..004574e3e9a7a 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -120,15 +120,3 @@ define <4 x half> @from_bf8_v4f16(<4 x i8> %x) {
%r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
ret <4 x half> %r
}
-
-; Constant input bypasses HW-specific path.
-define half @from_fp8_f16_const() {
-; CHECK-LABEL: from_fp8_f16_const:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_mov_b16_e32 v0.l, 0x3c00
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
- %r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 56, metadata !"Float8E4M3FN")
- ret half %r
-}
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index 200f5a43698b5..adc1184d7c5ea 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -260,50 +260,3 @@ define <3 x float> @v3_from_fp8(<3 x i8> %x) {
ret <3 x float> %r
}
-; Fall through to the generic expansion in case of constant inputs so they fold..
-define float @const_fp8() {
-; GFX950-LABEL: const_fp8:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1170-LABEL: const_fp8:
-; GFX1170: ; %bb.0:
-; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX1170-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: const_fp8:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
- %r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 56, metadata !"Float8E4M3FN")
- ret float %r
-}
-
-define <2 x float> @const_v2_fp8() {
-; GFX950-LABEL: const_v2_fp8:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX950-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1170-LABEL: const_v2_fp8:
-; GFX1170: ; %bb.0:
-; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
-; GFX1170-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: const_v2_fp8:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 2.0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
- %r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> <i8 56, i8 64>, metadata !"Float8E4M3FN")
- ret <2 x float> %r
-}
>From 70722cad1f044e23457f7de522e8d8729306544e Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 13 May 2026 18:55:45 +0200
Subject: [PATCH 10/12] remove RUN line
---
llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index 004574e3e9a7a..c1e1ad83cdda3 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck %s
-; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1251 | FileCheck %s
; llvm.convert.from.arbitrary.fp from FP8/BF8 producing f16 result types,
; lowered via the gfx1250+ unscaled v_cvt_{pk_}f16_{fp8,bf8} instructions.
>From f018a01feae5a939f4f6fc0aad174db423f7277d Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 13 May 2026 23:26:56 +0200
Subject: [PATCH 11/12] address
---
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 5 +++++
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 10 +++-------
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 90b5f7b0b794b..3e055a87aeb2c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -326,6 +326,11 @@ class UnscaledMFMAOptimizationPat<SDPatternOperator intrin> : PatFrag<
def mfma_f32_16x16x128_f8f6f4 : UnscaledMFMAOptimizationPat<int_amdgcn_mfma_scale_f32_16x16x128_f8f6f4>;
def mfma_f32_32x32x64_f8f6f4 : UnscaledMFMAOptimizationPat<int_amdgcn_mfma_scale_f32_32x32x64_f8f6f4>;
+def cvt_from_fp8 : PatFrag<(ops node:$src),
+ (convert_from_arbitrary_fp node:$src, Float8E4M3FN)>;
+def cvt_from_bf8 : PatFrag<(ops node:$src),
+ (convert_from_arbitrary_fp node:$src, Float8E5M2)>;
+
//===----------------------------------------------------------------------===//
// ValueType helpers
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 3a11678521d53..90f3cae7a96c6 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -711,11 +711,6 @@ class Cvt_F32_F8_Pat<SDPatternOperator node, int index,
(inst_sdwa 0, $src, 0, 0, index)
>;
-def cvt_from_fp8 : PatFrag<(ops node:$src),
- (convert_from_arbitrary_fp node:$src, Float8E4M3FN)>;
-def cvt_from_bf8 : PatFrag<(ops node:$src),
- (convert_from_arbitrary_fp node:$src, Float8E5M2)>;
-
let SubtargetPredicate = HasFP8ConversionInsts in {
let OtherPredicates = [HasCvtFP8VOP1Bug] in {
def : GCNPat<(f32 (int_amdgcn_cvt_f32_fp8 i32:$src, 0)),
@@ -738,7 +733,7 @@ foreach Index = [1, 2, 3] in {
}
} // End OtherPredicates = [HasSDWA]
-let OtherPredicates = [HasOCPFP8ConversionInsts, HasSDWA] in {
+let OtherPredicates = [HasOCPFP8ConversionInsts, isGFX9Only] in {
def : GCNPat<(f32 (cvt_from_fp8 i32:$src)), (V_CVT_F32_FP8_e32 $src)>;
def : GCNPat<(f32 (cvt_from_bf8 i32:$src)), (V_CVT_F32_BF8_e32 $src)>;
}
@@ -792,7 +787,8 @@ def V_CVT_F16_F8_True16_Profile : VOP3_Profile_True16<V_CVT_F16_F8_Profile>;
def V_CVT_F16_F8_Fake16_Profile : VOP3_Profile_Fake16<V_CVT_F16_F8_Profile>;
}
-let SubtargetPredicate = isGFX11Plus, OtherPredicates = [HasFP8ConversionInsts],
+let SubtargetPredicate = isGFX11Plus,
+ OtherPredicates = [HasFP8ConversionInsts, HasOCPFP8ConversionInsts],
mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
let SubtargetPredicate = isGFX11PlusNot12_50 in
defm V_CVT_F32_FP8_OP_SEL : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>, cvt_from_fp8>;
>From 118058b5701206ba4db4515e49c554ff0b5ca52b Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 11 Jun 2026 01:51:46 +0200
Subject: [PATCH 12/12] avoid Intrinsic::amdgcn during lowering
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 13 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 14 ++
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 21 +-
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 217 ++++++++++++------
4 files changed, 184 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index afa64a214dd2d..cc5a1d5d50843 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10779,12 +10779,9 @@ SDValue SITargetLowering::lowerFromFP8ToF32(SDValue Op, bool IsBF8,
SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f32_bf8
- : Intrinsic::amdgcn_cvt_pk_f32_fp8;
+ unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
SDValue PackedI32 = packBytesToI32(DAG, SL, Src);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f32,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI32,
- DAG.getTargetConstant(0, SL, MVT::i1));
+ return DAG.getNode(Opc, SL, MVT::v2f32, PackedI32);
}
SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
@@ -10793,12 +10790,10 @@ SDValue SITargetLowering::lowerFromFP8ToF16(SDValue Op, bool IsBF8,
"fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
- unsigned IntrID = IsBF8 ? Intrinsic::amdgcn_cvt_pk_f16_bf8
- : Intrinsic::amdgcn_cvt_pk_f16_fp8;
+ unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
SDValue PackedI32 = packBytesToI32(DAG, SL, Src);
SDValue PackedI16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, PackedI32);
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::v2f16,
- DAG.getTargetConstant(IntrID, SL, MVT::i32), PackedI16);
+ return DAG.getNode(Opc, SL, MVT::v2f16, PackedI16);
}
SDValue
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index f748ea6123bf5..1f398166899d8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -331,6 +331,20 @@ def cvt_from_fp8 : PatFrag<(ops node:$src),
def cvt_from_bf8 : PatFrag<(ops node:$src),
(convert_from_arbitrary_fp node:$src, Float8E5M2)>;
+def AMDGPUcvt_pk_f32_fp8 : SDNode<"AMDGPUISD::CVT_PK_F32_FP8",
+ SDTypeProfile<1, 1, [SDTCisVT<0, v2f32>, SDTCisVT<1, i32>]>>;
+def AMDGPUcvt_pk_f32_bf8 : SDNode<"AMDGPUISD::CVT_PK_F32_BF8",
+ SDTypeProfile<1, 1, [SDTCisVT<0, v2f32>, SDTCisVT<1, i32>]>>;
+def AMDGPUcvt_pk_f16_fp8 : SDNode<"AMDGPUISD::CVT_PK_F16_FP8",
+ SDTypeProfile<1, 1, [SDTCisVT<0, v2f16>, SDTCisVT<1, i16>]>>;
+def AMDGPUcvt_pk_f16_bf8 : SDNode<"AMDGPUISD::CVT_PK_F16_BF8",
+ SDTypeProfile<1, 1, [SDTCisVT<0, v2f16>, SDTCisVT<1, i16>]>>;
+
+def cvt_pk_f16_fp8 : PatFrags<(ops node:$src),
+ [(int_amdgcn_cvt_pk_f16_fp8 node:$src), (AMDGPUcvt_pk_f16_fp8 node:$src)]>;
+def cvt_pk_f16_bf8 : PatFrags<(ops node:$src),
+ [(int_amdgcn_cvt_pk_f16_bf8 node:$src), (AMDGPUcvt_pk_f16_bf8 node:$src)]>;
+
//===----------------------------------------------------------------------===//
// ValueType helpers
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 90f3cae7a96c6..aac789d5d86f4 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -748,7 +748,7 @@ class Cvt_PK_F32_F8_Pat<SDPatternOperator node, int index,
(inst_e32 $src))
>;
-let SubtargetPredicate = HasFP8ConversionInsts, OtherPredicates = [HasSDWA] in {
+let SubtargetPredicate = HasFP8ConversionInsts, OtherPredicates = [isGFX9Only] in {
foreach Index = [0, -1] in {
def : Cvt_PK_F32_F8_Pat<int_amdgcn_cvt_pk_f32_fp8, Index,
V_CVT_PK_F32_FP8_e32, V_CVT_PK_F32_FP8_sdwa>;
@@ -757,6 +757,16 @@ let SubtargetPredicate = HasFP8ConversionInsts, OtherPredicates = [HasSDWA] in {
}
}
+class Cvt_PK_F32_F8_Conv_Pat<SDPatternOperator node, Instruction inst> : GCNPat<
+ (v2f32 (node i32:$src)),
+ (inst $src)
+>;
+
+let SubtargetPredicate = HasFP8ConversionInsts, OtherPredicates = [isGFX9Only] in {
+ def : Cvt_PK_F32_F8_Conv_Pat<AMDGPUcvt_pk_f32_fp8, V_CVT_PK_F32_FP8_e32>;
+ def : Cvt_PK_F32_F8_Conv_Pat<AMDGPUcvt_pk_f32_bf8, V_CVT_PK_F32_BF8_e32>;
+}
+
let HasClamp = 0, HasOMod = 0, HasExtDPP = 0, HasExtVOP3DPP = 0,
HasOpSel = 1 in {
// Input modifiers are not supported
@@ -843,6 +853,11 @@ let SubtargetPredicate = isGFX11Plus, OtherPredicates = [HasFP8ConversionInsts]
}
}
+let SubtargetPredicate = isGFX11Plus, OtherPredicates = [HasFP8ConversionInsts] in {
+ def : Cvt_PK_F32_F8_Conv_Pat<AMDGPUcvt_pk_f32_fp8, V_CVT_PK_F32_FP8_fake16_e32>;
+ def : Cvt_PK_F32_F8_Conv_Pat<AMDGPUcvt_pk_f32_bf8, V_CVT_PK_F32_BF8_fake16_e32>;
+}
+
// FIXME-TRUE16: True16 versions of these instructions are untested.
let HasExtSDWA = 0, HasOpSel = 1, EmitDstSel = 0, HasOMod = 0, HasModifiers = 1 in {
def VOPProfile_CVT_PK_F16_F8 : VOPProfile<[v2f16, i16, untyped, untyped]>;
@@ -860,10 +875,10 @@ let SubtargetPredicate = HasFP8F16ConversionInsts in {
cvt_from_bf8>;
defm V_CVT_PK_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_fp8",
VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
- int_amdgcn_cvt_pk_f16_fp8>;
+ cvt_pk_f16_fp8>;
defm V_CVT_PK_F16_BF8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_bf8",
VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
- int_amdgcn_cvt_pk_f16_bf8>;
+ cvt_pk_f16_bf8>;
}
let True16Predicate = UseRealTrue16Insts in {
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index c1e1ad83cdda3..562f0cb8a9d76 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -1,121 +1,200 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 | FileCheck %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
+; RUN: llc < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
; llvm.convert.from.arbitrary.fp from FP8/BF8 producing f16 result types,
; lowered via the gfx1250+ unscaled v_cvt_{pk_}f16_{fp8,bf8} instructions.
; Scalar Float8E4M3FN
define half @from_fp8_f16_s(i8 %x) {
-; CHECK-LABEL: from_fp8_f16_s:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_fp8_f16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_fp8_f16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_cvt_f16_fp8_e32 v0, v0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E4M3FN")
ret half %r
}
; Scalar Float8E5M2
define half @from_bf8_f16_s(i8 %x) {
-; CHECK-LABEL: from_bf8_f16_s:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_bf8_f16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_bf8_f16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_cvt_f16_bf8_e32 v0, v0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %x, metadata !"Float8E5M2")
ret half %r
}
; Vector Float8E4M3FN
define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
-; CHECK-LABEL: from_fp8_v2f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_fp8_v2f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_fp8_v2f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
ret <2 x half> %r
}
; Vector Float8E5M2
define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
-; CHECK-LABEL: from_bf8_v2f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_bf8_v2f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_bf8_v2f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_bf8 v0, v0
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <2 x half> @llvm.convert.from.arbitrary.fp.v2f16.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
ret <2 x half> %r
}
; v3f16 Float8E4M3FN
define <3 x half> @from_fp8_v3f16(<3 x i8> %x) {
-; CHECK-LABEL: from_fp8_v3f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; CHECK-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_fp8_v3f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_fp8_v3f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_fp8 v1, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
ret <3 x half> %r
}
; v3f16 Float8E5M2
define <3 x half> @from_bf8_v3f16(<3 x i8> %x) {
-; CHECK-LABEL: from_bf8_v3f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
-; CHECK-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_bf8_v3f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_bf8_v3f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_bf8 v0, v0
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_bf8 v1, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <3 x half> @llvm.convert.from.arbitrary.fp.v3f16.v3i8(<3 x i8> %x, metadata !"Float8E5M2")
ret <3 x half> %r
}
; v4f16 Float8E4M3FN
define <4 x half> @from_fp8_v4f16(<4 x i8> %x) {
-; CHECK-LABEL: from_fp8_v4f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
-; CHECK-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_fp8_v4f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v1, v1.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_fp8_v4f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_fp8 v1, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
ret <4 x half> %r
}
; v4f16 Float8E5M2
define <4 x half> @from_bf8_v4f16(<4 x i8> %x) {
-; CHECK-LABEL: from_bf8_v4f16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; CHECK-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
-; CHECK-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
-; CHECK-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-TRUE16-LABEL: from_bf8_v4f16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v1, v1.l
+; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: from_bf8_v4f16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX1250-FAKE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_bf8 v0, v0
+; GFX1250-FAKE16-NEXT: v_cvt_pk_f16_bf8 v1, v1
+; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x half> @llvm.convert.from.arbitrary.fp.v4f16.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
ret <4 x half> %r
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1250: {{.*}}
More information about the llvm-commits
mailing list