[llvm] 755e4d4 - [SelectionDAG] Fix FP8 conversions to half for subnormals and overflow (#224984)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 13:46:46 PDT 2026
Author: Chinmay Deshpande
Date: 2026-09-22T13:46:40-07:00
New Revision: 755e4d429f11a17ba386cdf4d4ff7883294368dd
URL: https://github.com/llvm/llvm-project/commit/755e4d429f11a17ba386cdf4d4ff7883294368dd
DIFF: https://github.com/llvm/llvm-project/commit/755e4d429f11a17ba386cdf4d4ff7883294368dd.diff
LOG: [SelectionDAG] Fix FP8 conversions to half for subnormals and overflow (#224984)
This PR contains 2 fixes:
* Preserve subnormal values when the source format and destination have
the same exponent bias. Normalizing these inputs previously produced a
nonpositive destination exponent and an invalid floating-point encoding.
* Convert finite values beyond the destination exponent range to
infinity, while preserving the existing NaN handling. This fixes
Float8E5M3FNU to half conversions that previously produced NaNs on
overflow.
Assisted by: Codex (Astra)
Added:
llvm/test/CodeGen/X86/arbitrary-fp-to-half.ll
Modified:
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7b92c903a209e..7c97dfb2c806a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9879,9 +9879,10 @@ TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
DAG.getNode(ISD::OR, dl, IntVT, SignShifted, NormExpShifted),
NormDstMant);
- // Denormal value conversion.
- SDValue DenormResult;
- {
+ // With identical exponent biases, denormal values remain denormal and the
+ // normal conversion's mantissa shift is sufficient.
+ SDValue DenormResult = NormResult;
+ if (BiasAdjust != 0) {
const unsigned IntVTBits = IntVT.getScalarSizeInBits();
SDValue LeadingZeros =
DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, IntVT, MantField);
@@ -9925,6 +9926,18 @@ TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
DAG.getConstant(DstExpAllOnes << DstMant, dl, IntVT));
+ // A source format may have a larger finite exponent range despite having
+ // fewer bits, as with Float8E5M3FNU converted to half. Its overflowing finite
+ // values become infinity. The NaN selection below still takes precedence.
+ if (APFloat::semanticsMaxExponent(SrcSem) >
+ APFloat::semanticsMaxExponent(DstSem)) {
+ SDValue IsOverflow =
+ DAG.getSetCC(dl, SetCCVT, NormDstExp,
+ DAG.getConstant(DstExpAllOnes, dl, IntVT), ISD::SETUGE);
+ FiniteResult =
+ DAG.getSelect(dl, IntVT, IsOverflow, InfResult, FiniteResult);
+ }
+
SDValue ZeroResult = SignShifted;
SDValue Result = FiniteResult;
diff --git a/llvm/test/CodeGen/X86/arbitrary-fp-to-half.ll b/llvm/test/CodeGen/X86/arbitrary-fp-to-half.ll
new file mode 100644
index 0000000000000..8c52b7057b0c3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/arbitrary-fp-to-half.ll
@@ -0,0 +1,208 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512fp16,+avx512vl | FileCheck %s
+
+; These source formats and half share a minimum normal exponent of -14.
+; Subnormal inputs must therefore remain subnormal in half. Return the half
+; encodings as integers so their sign, exponent and mantissa are visible.
+; Enable AVX512-FP16 to exercise vector expansion without scalarizing half.
+
+declare half @llvm.convert.from.arbitrary.fp.f16.i8(i8, metadata)
+declare <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(<8 x i8>, metadata)
+
+; All signed E5M2 subnormals and both signed zeros.
+define <8 x i16> @e5m2_subnormals() {
+; CHECK-LABEL: e5m2_subnormals:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [0,256,512,768,32768,33024,33280,33536]
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> <i8 0, i8 1, i8 2, i8 3, i8 128, i8 129, i8 130, i8 131>,
+ metadata !"Float8E5M2")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+; All seven unsigned E5M3FNU subnormals and zero.
+define <8 x i16> @e5m3fnu_subnormals() {
+; CHECK-LABEL: e5m3fnu_subnormals:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [0,128,256,384,512,640,768,896]
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>,
+ metadata !"Float8E5M3FNU")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+; Largest subnormal, smallest normal, largest finite, and infinity, both signs.
+define <8 x i16> @e5m2_boundaries() {
+; CHECK-LABEL: e5m2_boundaries:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [768,1024,31488,31744,33536,33792,64256,64512]
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> <i8 3, i8 4, i8 123, i8 124, i8 131, i8 132, i8 251, i8 252>,
+ metadata !"Float8E5M2")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+; 61440 is representable in half. The next seven finite encodings, 65536
+; through 114688, overflow to infinity rather than producing NaNs.
+define <8 x i16> @e5m3fnu_overflow() {
+; CHECK-LABEL: e5m3fnu_overflow:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [31616,31744,31744,31744,31744,31744,31744,31744]
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> <i8 247, i8 248, i8 249, i8 250, i8 251, i8 252, i8 253, i8 254>,
+ metadata !"Float8E5M3FNU")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+; Include the normal/subnormal boundary, the unsigned exponent's top bit, and
+; the NaN encoding, which must take precedence over the overflow check.
+define <8 x i16> @e5m3fnu_boundaries() {
+; CHECK-LABEL: e5m3fnu_boundaries:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [896,1024,1152,16256,16384,31616,31744,32256]
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> <i8 7, i8 8, i8 9, i8 127, i8 128, i8 247, i8 248, i8 255>,
+ metadata !"Float8E5M3FNU")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+define i16 @e5m2_scalar(i8 %src) {
+; CHECK-LABEL: e5m2_scalar:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %ecx
+; CHECK-NEXT: shll $8, %ecx
+; CHECK-NEXT: andl $32768, %ecx # imm = 0x8000
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: shrl $2, %eax
+; CHECK-NEXT: andl $31, %eax
+; CHECK-NEXT: movl %eax, %edx
+; CHECK-NEXT: shll $10, %edx
+; CHECK-NEXT: orl %ecx, %edx
+; CHECK-NEXT: andl $3, %edi
+; CHECK-NEXT: movl %edi, %esi
+; CHECK-NEXT: shll $8, %esi
+; CHECK-NEXT: orl %edx, %esi
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: setne %dl
+; CHECK-NEXT: sete %dil
+; CHECK-NEXT: testl %eax, %eax
+; CHECK-NEXT: sete %r8b
+; CHECK-NEXT: testb %dil, %r8b
+; CHECK-NEXT: cmovnel %ecx, %esi
+; CHECK-NEXT: orl $31744, %ecx # imm = 0x7C00
+; CHECK-NEXT: cmpw $31, %ax
+; CHECK-NEXT: sete %al
+; CHECK-NEXT: testb %dil, %al
+; CHECK-NEXT: cmovel %esi, %ecx
+; CHECK-NEXT: testb %dl, %al
+; CHECK-NEXT: movl $32256, %eax # imm = 0x7E00
+; CHECK-NEXT: cmovel %ecx, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %fp = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %src, metadata !"Float8E5M2")
+ %bits = bitcast half %fp to i16
+ ret i16 %bits
+}
+
+define i16 @e5m3fnu_scalar(i8 %src) {
+; CHECK-LABEL: e5m3fnu_scalar:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: andl $7, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: shll $7, %ecx
+; CHECK-NEXT: shrl $3, %edi
+; CHECK-NEXT: andl $31, %edi
+; CHECK-NEXT: movl %edi, %edx
+; CHECK-NEXT: shll $10, %edx
+; CHECK-NEXT: orl %ecx, %edx
+; CHECK-NEXT: cmpw $31, %di
+; CHECK-NEXT: movl $31744, %ecx # imm = 0x7C00
+; CHECK-NEXT: cmovbl %edx, %ecx
+; CHECK-NEXT: sete %dl
+; CHECK-NEXT: testl %eax, %eax
+; CHECK-NEXT: sete %sil
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sete %dil
+; CHECK-NEXT: xorl %r8d, %r8d
+; CHECK-NEXT: testb %sil, %dil
+; CHECK-NEXT: cmovel %ecx, %r8d
+; CHECK-NEXT: cmpw $7, %ax
+; CHECK-NEXT: sete %al
+; CHECK-NEXT: testb %al, %dl
+; CHECK-NEXT: movl $32256, %eax # imm = 0x7E00
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %fp = call half @llvm.convert.from.arbitrary.fp.f16.i8(i8 %src, metadata !"Float8E5M3FNU")
+ %bits = bitcast half %fp to i16
+ ret i16 %bits
+}
+
+define <8 x i16> @e5m2_vector(<8 x i8> %src) {
+; CHECK-LABEL: e5m2_vector:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
+; CHECK-NEXT: vpsllw $8, %xmm2, %xmm3
+; CHECK-NEXT: vpsrlw $2, %ymm1, %ymm0
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm4 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; CHECK-NEXT: vpand %ymm4, %ymm0, %ymm5
+; CHECK-NEXT: vpsllw $10, %xmm5, %xmm0
+; CHECK-NEXT: vpsrlw $7, %ymm1, %ymm1
+; CHECK-NEXT: vpsllw $15, %ymm1, %ymm1
+; CHECK-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | ymm1 | ymm3
+; CHECK-NEXT: vpor %ymm2, %ymm5, %ymm3
+; CHECK-NEXT: vptestnmw %ymm3, %ymm3, %k1
+; CHECK-NEXT: vmovdqu16 %ymm1, %ymm0 {%k1}
+; CHECK-NEXT: vpcmpeqw %ymm4, %ymm5, %k1
+; CHECK-NEXT: vptestnmw %ymm2, %ymm2, %k2 {%k1}
+; CHECK-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; CHECK-NEXT: vmovdqu16 %ymm1, %ymm0 {%k2}
+; CHECK-NEXT: vptestmw %ymm2, %ymm2, %k1 {%k1}
+; CHECK-NEXT: vmovdqu16 {{.*#+}} ymm0 {%k1} = [32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> %src, metadata !"Float8E5M2")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
+
+define <8 x i16> @e5m3fnu_vector(<8 x i8> %src) {
+; CHECK-LABEL: e5m3fnu_vector:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; CHECK-NEXT: vpand %ymm1, %ymm0, %ymm2
+; CHECK-NEXT: vpsllw $7, %xmm2, %xmm3
+; CHECK-NEXT: vpsrlw $3, %ymm0, %ymm4
+; CHECK-NEXT: vpsllw $10, %xmm4, %xmm0
+; CHECK-NEXT: vpor %xmm3, %xmm0, %xmm0
+; CHECK-NEXT: vpcmpgtw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %k1
+; CHECK-NEXT: vmovdqu16 {{.*#+}} ymm0 {%k1} = [31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744,31744]
+; CHECK-NEXT: vpor %ymm2, %ymm4, %ymm3
+; CHECK-NEXT: vptestmw %ymm3, %ymm3, %k1
+; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpcmpeqw %ymm1, %ymm2, %k1
+; CHECK-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %k1 {%k1}
+; CHECK-NEXT: vmovdqu16 {{.*#+}} ymm0 {%k1} = [32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256,32256]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %fp = call <8 x half> @llvm.convert.from.arbitrary.fp.v8f16.v8i8(
+ <8 x i8> %src, metadata !"Float8E5M3FNU")
+ %bits = bitcast <8 x half> %fp to <8 x i16>
+ ret <8 x i16> %bits
+}
More information about the llvm-commits
mailing list