[llvm] 82442a5 - [AArch64] Fix legalization of bf16 ldexp. (#190805)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 00:37:04 PDT 2026
Author: David Green
Date: 2026-04-13T08:36:58+01:00
New Revision: 82442a58c054611219385528f06a05bf1eb19215
URL: https://github.com/llvm/llvm-project/commit/82442a58c054611219385528f06a05bf1eb19215
DIFF: https://github.com/llvm/llvm-project/commit/82442a58c054611219385528f06a05bf1eb19215.diff
LOG: [AArch64] Fix legalization of bf16 ldexp. (#190805)
Similar to fp16 ldexp, we cannot create illegal types for bf16 during
lowering so should promote.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/Analysis/CostModel/AArch64/ldexp.ll
llvm/test/CodeGen/AArch64/ldexp.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4b6b0758006af..96ccd38e34117 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2164,11 +2164,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(Op, MVT::f32, Promote);
}
- // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
- // isn't legal.
- for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
+ // LegalizeDAG currently can't expand fp16/bf16 LDEXP/FREXP on targets where
+ // i16 isn't legal.
+ for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP}) {
if (isOperationExpand(Op, MVT::f16))
setOperationAction(Op, MVT::f16, Promote);
+ if (isOperationExpand(Op, MVT::bf16))
+ setOperationAction(Op, MVT::bf16, Promote);
+ }
}
const AArch64TargetMachine &AArch64TargetLowering::getTM() const {
diff --git a/llvm/test/Analysis/CostModel/AArch64/ldexp.ll b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
index 8cc8657468930..60ab7bc964680 100644
--- a/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
@@ -72,11 +72,11 @@ define void @ldexp_fp16() {
define void @ldexp_bf16() {
; CHECK-BASE-LABEL: 'ldexp_bf16'
-; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
-; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
-; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:46 CodeSize:7 Lat:46 SizeLat:46 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
-; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:94 CodeSize:15 Lat:94 SizeLat:94 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
-; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:188 CodeSize:30 Lat:188 SizeLat:188 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT: Cost Model: Found costs of 1 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:10 CodeSize:7 Lat:10 SizeLat:10 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:15 Lat:22 SizeLat:22 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:44 CodeSize:30 Lat:44 SizeLat:44 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
; CHECK-BASE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-SVE-LABEL: 'ldexp_bf16'
diff --git a/llvm/test/CodeGen/AArch64/ldexp.ll b/llvm/test/CodeGen/AArch64/ldexp.ll
index 2de20c294dcb3..308e861abb6d4 100644
--- a/llvm/test/CodeGen/AArch64/ldexp.ll
+++ b/llvm/test/CodeGen/AArch64/ldexp.ll
@@ -1,9 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -mtriple=aarch64 -mattr=+sve < %s -o - | FileCheck -check-prefixes=SVE,SVELINUX %s
-; RUN: llc -mtriple=aarch64 -global-isel < %s -o - | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 < %s -o - 2>&1 | FileCheck -check-prefixes=GISEL %s
; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+sve < %s -o - | FileCheck -check-prefixes=SVE,SVEWINDOWS %s
; RUN: llc -mtriple=aarch64-windows-msvc < %s -o - | FileCheck -check-prefixes=WINDOWS %s
+; GISEL: warning: Instruction selection used fallback path for testExpbf16
+
define double @testExp(double %val, i32 %a) {
; SVE-LABEL: testExp:
; SVE: // %bb.0: // %entry
@@ -263,4 +265,70 @@ entry:
ret half %0
}
-declare half @llvm.ldexp.f16.i32(half, i32) memory(none)
+define bfloat @testExpbf16(bfloat %val, i32 %a) {
+; SVE-LABEL: testExpbf16:
+; SVE: // %bb.0: // %entry
+; SVE-NEXT: // kill: def $h0 killed $h0 def $d0
+; SVE-NEXT: fmov s1, w0
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: shll v0.4s, v0.4h, #16
+; SVE-NEXT: fscale z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: fmov w8, s0
+; SVE-NEXT: fcmp s0, s0
+; SVE-NEXT: orr w9, w8, #0x400000
+; SVE-NEXT: csel w8, w9, w8, vs
+; SVE-NEXT: lsr w8, w8, #16
+; SVE-NEXT: fmov s0, w8
+; SVE-NEXT: // kill: def $h0 killed $h0 killed $s0
+; SVE-NEXT: ret
+;
+; GISEL-LABEL: testExpbf16:
+; GISEL: // %bb.0: // %entry
+; GISEL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; GISEL-NEXT: .cfi_def_cfa_offset 16
+; GISEL-NEXT: .cfi_offset w30, -16
+; GISEL-NEXT: // kill: def $h0 killed $h0 def $d0
+; GISEL-NEXT: shll v0.4s, v0.4h, #16
+; GISEL-NEXT: // kill: def $s0 killed $s0 killed $q0
+; GISEL-NEXT: bl ldexpf
+; GISEL-NEXT: fmov w9, s0
+; GISEL-NEXT: mov w8, #32767 // =0x7fff
+; GISEL-NEXT: ubfx w10, w9, #16, #1
+; GISEL-NEXT: add w8, w9, w8
+; GISEL-NEXT: add w8, w10, w8
+; GISEL-NEXT: lsr w8, w8, #16
+; GISEL-NEXT: fmov s0, w8
+; GISEL-NEXT: // kill: def $h0 killed $h0 killed $s0
+; GISEL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; GISEL-NEXT: ret
+;
+; WINDOWS-LABEL: testExpbf16:
+; WINDOWS: .seh_proc testExpbf16
+; WINDOWS-NEXT: // %bb.0: // %entry
+; WINDOWS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT: .seh_save_reg_x x30, 16
+; WINDOWS-NEXT: .seh_endprologue
+; WINDOWS-NEXT: // kill: def $h0 killed $h0 def $d0
+; WINDOWS-NEXT: shll v0.4s, v0.4h, #16
+; WINDOWS-NEXT: fcvt d0, s0
+; WINDOWS-NEXT: bl ldexp
+; WINDOWS-NEXT: fcvtxn s0, d0
+; WINDOWS-NEXT: mov w8, #32767 // =0x7fff
+; WINDOWS-NEXT: fmov w9, s0
+; WINDOWS-NEXT: ubfx w10, w9, #16, #1
+; WINDOWS-NEXT: add w8, w9, w8
+; WINDOWS-NEXT: add w8, w10, w8
+; WINDOWS-NEXT: lsr w8, w8, #16
+; WINDOWS-NEXT: fmov s0, w8
+; WINDOWS-NEXT: // kill: def $h0 killed $h0 killed $s0
+; WINDOWS-NEXT: .seh_startepilogue
+; WINDOWS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT: .seh_save_reg_x x30, 16
+; WINDOWS-NEXT: .seh_endepilogue
+; WINDOWS-NEXT: ret
+; WINDOWS-NEXT: .seh_endfunclet
+; WINDOWS-NEXT: .seh_endproc
+entry:
+ %0 = tail call fast bfloat @llvm.ldexp.bf16.i32(bfloat %val, i32 %a)
+ ret bfloat %0
+}
More information about the llvm-commits
mailing list