[llvm] [AMDGPU] Legalize strict fp_extend from bf16 (PR #215477)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 11:06:46 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215477
>From 5b687639d5daa75067a60ba6084e39d594216c85 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 08:39:39 +0200
Subject: [PATCH 1/3] [AMDGPU] Legalize strict fp_extend from bf16
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 ++-
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 53 ++++++++++++++++++++
2 files changed, 58 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fb5c4279af4b7..c3316f3c0b33c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5006,10 +5006,13 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
EVT DstVT = Op.getValueType();
+ SDValue Result = DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
+ // bf16 -> f32/f64 extension is exact and cannot raise an FP exception, so
+ // the strict chain can pass through unchanged.
if (IsStrict)
- llvm_unreachable("Need STRICT_BF16_TO_FP");
+ return DAG.getMergeValues({Result, Op.getOperand(0)}, SL);
- return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
+ return Result;
}
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 4ce105849772c..306ab81c84075 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -43,6 +43,55 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
ret float %cvt
}
+define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
+; GCN-LABEL: strict_fpext_bf16_to_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v3, v2
+; GCN-NEXT: v_mov_b32_e32 v2, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: flat_store_dword v[2:3], v0
+; GCN-NEXT: s_endpgm
+;
+; GFX1250-LABEL: strict_fpext_bf16_to_f32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: flat_store_b32 v[2:3], v0
+; GFX1250-NEXT: s_endpgm
+ %cvt = call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %a, metadata !"fpexcept.strict")
+ store float %cvt, ptr %out
+ ret void
+}
+
+define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
+; GCN-LABEL: strict_fpext_bf16_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_mov_b32_e32 v3, v2
+; GCN-NEXT: v_mov_b32_e32 v2, v1
+; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
+;
+; GFX1250-LABEL: strict_fpext_bf16_to_f64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b32_e32 v2, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX1250-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX1250-NEXT: s_endpgm
+ %cvt = call double @llvm.experimental.constrained.fpext.f64.bf16(bfloat %a, metadata !"fpexcept.strict")
+ store double %cvt, ptr %out
+ ret void
+}
+
define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) {
; GFX-942-LABEL: v_test_cvt_v2f32_v2bf16_v:
; GFX-942: ; %bb.0:
@@ -661,6 +710,10 @@ entry:
declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)
+declare float @llvm.experimental.constrained.fpext.f32.bf16(bfloat, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.bf16(bfloat, metadata)
+
+attributes #0 = { strictfp }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1250-FAKE16: {{.*}}
; GFX1250-REAL16: {{.*}}
>From 06e9be724999b05ab247c1d4734ac43ce0dcef1a Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 14:22:56 +0200
Subject: [PATCH 2/3] Address comments
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +++++++-----
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 12 +++++++++---
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c3316f3c0b33c..b6218174d6127 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5007,12 +5007,14 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
EVT DstVT = Op.getValueType();
SDValue Result = DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
- // bf16 -> f32/f64 extension is exact and cannot raise an FP exception, so
- // the strict chain can pass through unchanged.
- if (IsStrict)
- return DAG.getMergeValues({Result, Op.getOperand(0)}, SL);
+ if (!IsStrict)
+ return Result;
- return Result;
+ // Route through a strict add of -0.0, exact for every input including
+ // sign of zero, so a real FP instruction quiets/traps on a signaling NaN.
+ SDValue NegZero = DAG.getConstantFP(-0.0, SL, DstVT);
+ return DAG.getNode(ISD::STRICT_FADD, SL, DAG.getVTList(DstVT, MVT::Other),
+ {Op.getOperand(0), Result, NegZero});
}
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 306ab81c84075..cffeb5e9301dd 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -46,9 +46,10 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GCN-LABEL: strict_fpext_bf16_to_f32:
; GCN: ; %bb.0:
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_add_f32_e32 v0, 0x80000000, v0
; GCN-NEXT: flat_store_dword v[2:3], v0
; GCN-NEXT: s_endpgm
;
@@ -57,8 +58,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_brev_b32 s0, 1
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
; GFX1250-NEXT: flat_store_b32 v[2:3], v0
; GFX1250-NEXT: s_endpgm
%cvt = call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %a, metadata !"fpexcept.strict")
@@ -73,6 +75,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_brev_b32 s1, 1
+; GCN-NEXT: v_add_f64 v[0:1], v[0:1], s[0:1]
; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GCN-NEXT: s_endpgm
;
@@ -83,8 +88,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX1250-NEXT: v_add_f64_e32 v[0:1], 0x80000000, v[0:1]
; GFX1250-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX1250-NEXT: s_endpgm
%cvt = call double @llvm.experimental.constrained.fpext.f64.bf16(bfloat %a, metadata !"fpexcept.strict")
>From 594a34181c986195e95fdc0fc92b65c0b06796a8 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 19 Aug 2026 20:06:32 +0200
Subject: [PATCH 3/3] Address comments
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 30 ++++++++++++++------
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 12 ++------
3 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b6218174d6127..f39285060c068 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -600,6 +600,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// source bf16.
setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f32, Custom);
setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f64, Custom);
+ setOperationAction(ISD::STRICT_BF16_TO_FP, MVT::f32, Custom);
+ setOperationAction(ISD::STRICT_BF16_TO_FP, MVT::f64, Expand);
if (Subtarget->has16BitInsts()) {
setOperationAction({ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
@@ -5006,15 +5008,25 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
EVT DstVT = Op.getValueType();
- SDValue Result = DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
- if (!IsStrict)
- return Result;
+ if (IsStrict)
+ return DAG.getNode(ISD::STRICT_BF16_TO_FP, SL, Op->getVTList(),
+ {Op.getOperand(0), BitCast});
+
+ return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
+}
+
+SDValue SITargetLowering::lowerSTRICT_BF16_TO_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Op.getValueType() == MVT::f32);
+ SDLoc SL(Op);
+ SDValue Chain = Op.getOperand(0);
+ SDValue Src = Op.getOperand(1);
- // Route through a strict add of -0.0, exact for every input including
- // sign of zero, so a real FP instruction quiets/traps on a signaling NaN.
- SDValue NegZero = DAG.getConstantFP(-0.0, SL, DstVT);
- return DAG.getNode(ISD::STRICT_FADD, SL, DAG.getVTList(DstVT, MVT::Other),
- {Op.getOperand(0), Result, NegZero});
+ SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
+ SDValue Shifted = DAG.getNode(ISD::SHL, SL, MVT::i32, Ext,
+ DAG.getShiftAmountConstant(16, MVT::i32, SL));
+ SDValue F32 = DAG.getNode(ISD::BITCAST, SL, MVT::f32, Shifted);
+ return DAG.getMergeValues({F32, Chain}, SL);
}
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
@@ -7792,6 +7804,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::FP_EXTEND:
case ISD::STRICT_FP_EXTEND:
return lowerFP_EXTEND(Op, DAG);
+ case ISD::STRICT_BF16_TO_FP:
+ return lowerSTRICT_BF16_TO_FP(Op, DAG);
case ISD::GET_FPENV:
return lowerGET_FPENV(Op, DAG);
case ISD::SET_FPENV:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 64d71f09edc33..fe37c58c2350c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -475,6 +475,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerSTRICT_BF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerROTR(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index cffeb5e9301dd..306ab81c84075 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -46,10 +46,9 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GCN-LABEL: strict_fpext_bf16_to_f32:
; GCN: ; %bb.0:
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: v_add_f32_e32 v0, 0x80000000, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: flat_store_dword v[2:3], v0
; GCN-NEXT: s_endpgm
;
@@ -58,9 +57,8 @@ define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_brev_b32 s0, 1
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-NEXT: flat_store_b32 v[2:3], v0
; GFX1250-NEXT: s_endpgm
%cvt = call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %a, metadata !"fpexcept.strict")
@@ -75,9 +73,6 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GCN-NEXT: s_mov_b32 s0, 0
-; GCN-NEXT: s_brev_b32 s1, 1
-; GCN-NEXT: v_add_f64 v[0:1], v[0:1], s[0:1]
; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GCN-NEXT: s_endpgm
;
@@ -88,9 +83,8 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: v_add_f64_e32 v[0:1], 0x80000000, v[0:1]
; GFX1250-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX1250-NEXT: s_endpgm
%cvt = call double @llvm.experimental.constrained.fpext.f64.bf16(bfloat %a, metadata !"fpexcept.strict")
More information about the llvm-commits
mailing list