[llvm] [AMDGPU] Legalize strict fp_extend from bf16 (PR #215477)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 05:23:09 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215477
>From 5b687639d5daa75067a60ba6084e39d594216c85 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 08:39:39 +0200
Subject: [PATCH 1/2] [AMDGPU] Legalize strict fp_extend from bf16
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 ++-
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 53 ++++++++++++++++++++
2 files changed, 58 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fb5c4279af4b7..c3316f3c0b33c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5006,10 +5006,13 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
EVT DstVT = Op.getValueType();
+ SDValue Result = DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
+ // bf16 -> f32/f64 extension is exact and cannot raise an FP exception, so
+ // the strict chain can pass through unchanged.
if (IsStrict)
- llvm_unreachable("Need STRICT_BF16_TO_FP");
+ return DAG.getMergeValues({Result, Op.getOperand(0)}, SL);
- return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
+ return Result;
}
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 4ce105849772c..306ab81c84075 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -43,6 +43,55 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
ret float %cvt
}
+define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
+; GCN-LABEL: strict_fpext_bf16_to_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_mov_b32_e32 v3, v2
+; GCN-NEXT: v_mov_b32_e32 v2, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: flat_store_dword v[2:3], v0
+; GCN-NEXT: s_endpgm
+;
+; GFX1250-LABEL: strict_fpext_bf16_to_f32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: flat_store_b32 v[2:3], v0
+; GFX1250-NEXT: s_endpgm
+ %cvt = call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %a, metadata !"fpexcept.strict")
+ store float %cvt, ptr %out
+ ret void
+}
+
+define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
+; GCN-LABEL: strict_fpext_bf16_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_mov_b32_e32 v3, v2
+; GCN-NEXT: v_mov_b32_e32 v2, v1
+; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
+;
+; GFX1250-LABEL: strict_fpext_bf16_to_f64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b32_e32 v2, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX1250-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX1250-NEXT: s_endpgm
+ %cvt = call double @llvm.experimental.constrained.fpext.f64.bf16(bfloat %a, metadata !"fpexcept.strict")
+ store double %cvt, ptr %out
+ ret void
+}
+
define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) {
; GFX-942-LABEL: v_test_cvt_v2f32_v2bf16_v:
; GFX-942: ; %bb.0:
@@ -661,6 +710,10 @@ entry:
declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)
+declare float @llvm.experimental.constrained.fpext.f32.bf16(bfloat, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.bf16(bfloat, metadata)
+
+attributes #0 = { strictfp }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1250-FAKE16: {{.*}}
; GFX1250-REAL16: {{.*}}
>From 06e9be724999b05ab247c1d4734ac43ce0dcef1a Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 14:22:56 +0200
Subject: [PATCH 2/2] Address comments
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +++++++-----
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 12 +++++++++---
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c3316f3c0b33c..b6218174d6127 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5007,12 +5007,14 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
EVT DstVT = Op.getValueType();
SDValue Result = DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
- // bf16 -> f32/f64 extension is exact and cannot raise an FP exception, so
- // the strict chain can pass through unchanged.
- if (IsStrict)
- return DAG.getMergeValues({Result, Op.getOperand(0)}, SL);
+ if (!IsStrict)
+ return Result;
- return Result;
+ // Route through a strict add of -0.0, exact for every input including
+ // sign of zero, so a real FP instruction quiets/traps on a signaling NaN.
+ SDValue NegZero = DAG.getConstantFP(-0.0, SL, DstVT);
+ return DAG.getNode(ISD::STRICT_FADD, SL, DAG.getVTList(DstVT, MVT::Other),
+ {Op.getOperand(0), Result, NegZero});
}
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 306ab81c84075..cffeb5e9301dd 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -46,9 +46,10 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GCN-LABEL: strict_fpext_bf16_to_f32:
; GCN: ; %bb.0:
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_add_f32_e32 v0, 0x80000000, v0
; GCN-NEXT: flat_store_dword v[2:3], v0
; GCN-NEXT: s_endpgm
;
@@ -57,8 +58,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f32(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_brev_b32 s0, 1
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
; GFX1250-NEXT: flat_store_b32 v[2:3], v0
; GFX1250-NEXT: s_endpgm
%cvt = call float @llvm.experimental.constrained.fpext.f32.bf16(bfloat %a, metadata !"fpexcept.strict")
@@ -73,6 +75,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GCN-NEXT: v_mov_b32_e32 v3, v2
; GCN-NEXT: v_mov_b32_e32 v2, v1
; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_brev_b32 s1, 1
+; GCN-NEXT: v_add_f64 v[0:1], v[0:1], s[0:1]
; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GCN-NEXT: s_endpgm
;
@@ -83,8 +88,9 @@ define amdgpu_ps void @strict_fpext_bf16_to_f64(bfloat %a, ptr %out) #0 {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX1250-NEXT: v_add_f64_e32 v[0:1], 0x80000000, v[0:1]
; GFX1250-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX1250-NEXT: s_endpgm
%cvt = call double @llvm.experimental.constrained.fpext.f64.bf16(bfloat %a, metadata !"fpexcept.strict")
More information about the llvm-commits
mailing list