[llvm] [AArch64][GlobalISel] Legalize F64 to BF16 fptruncates (PR #196077)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 06:50:24 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/196077
This two-step expansion of bf16 fptrunc steps needs to be careful to avoid double-rounding error. Under AArch64 we can apparently convert to a fcvtxn that performs round-to-odd, followed by a standard fp truncate to bf16 to make sure the rounding from there is done correctly. This reuses the existing lowering added for vector operations.
>From b4bb31af14c8d50ddc83ad930deda97eec81c347 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 6 May 2026 14:45:56 +0100
Subject: [PATCH] [AArch64][GlobalISel] Legalize F64 to BF16 fptruncates
This two-step expansion of bf16 fptrunc steps needs to be careful to avoid
double-rounding error. Under AArch64 we can apparently convert to a fcvtxn that
performs round-to-odd, followed by a standard fp truncate to bf16 to make sure
the rounding from there is done correctly. This reuses the existing lowering
added for vector operations.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 14 ++-
.../AArch64/GISel/AArch64RegisterBankInfo.cpp | 2 +
llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll | 11 +-
.../test/CodeGen/AArch64/bf16-instructions.ll | 115 ++++++++++++------
.../CodeGen/AArch64/bf16-v4-instructions.ll | 50 +++++---
.../CodeGen/AArch64/bf16-v8-instructions.ll | 80 ++++++++----
6 files changed, 184 insertions(+), 88 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e8bc74a150b57..4a03f80db3e56 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -872,9 +872,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.customIf([](const LegalityQuery &Q) {
LLT DstTy = Q.Types[0];
LLT SrcTy = Q.Types[1];
- return SrcTy.isFixedVector() && DstTy.isFixedVector() &&
- SrcTy.getScalarType().isFloat64() &&
- DstTy.getScalarType().isFloat16();
+ return SrcTy.getScalarSizeInBits() == 64 &&
+ DstTy.getScalarSizeInBits() == 16;
})
.lowerFor({{bf16, f32}, {v4bf16, v4f32}})
// Clamp based on input
@@ -2659,6 +2658,15 @@ bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
MachineIRBuilder &MIRBuilder,
MachineRegisterInfo &MRI) const {
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+
+ if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
+ auto Mid =
+ MIRBuilder.buildInstr(AArch64::G_FPTRUNC_ODD, {LLT::float32()}, {Src});
+ MIRBuilder.buildInstr(AArch64::G_FPTRUNC, {Dst}, {Mid}).getReg(0);
+ MI.eraseFromParent();
+ return true;
+ }
+
assert(SrcTy.isFixedVector() && isPowerOf2_32(SrcTy.getNumElements()) &&
"Expected a power of 2 elements");
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 947a3a5ea170b..eb6d42839056b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -733,6 +733,7 @@ bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
case AArch64::G_PMULL:
case AArch64::G_SLI:
case AArch64::G_SRI:
+ case AArch64::G_FPTRUNC_ODD:
return true;
case TargetOpcode::G_INTRINSIC:
switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
@@ -773,6 +774,7 @@ bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
case TargetOpcode::G_BUILD_VECTOR_TRUNC:
case AArch64::G_SLI:
case AArch64::G_SRI:
+ case AArch64::G_FPTRUNC_ODD:
return true;
case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
index 94b494c8c08c4..b2d9237531900 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -3,7 +3,6 @@
; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -O0 -fast-isel | FileCheck %s --check-prefixes=CHECK,CHECK-FI
; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for test_vcvt_bf16_f64
define <2 x double> @test_vcvt_f64_f32(<2 x float> %x) nounwind readnone ssp {
; CHECK-LABEL: test_vcvt_f64_f32:
@@ -172,13 +171,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp {
; CHECK-GI-NEXT: fcvtxn v0.2s, v0.2d
; CHECK-GI-NEXT: movi.4s v1, #1
; CHECK-GI-NEXT: movi.4s v2, #127, msl #8
+; CHECK-GI-NEXT: movi.4s v5, #64, lsl #16
; CHECK-GI-NEXT: ushr.4s v3, v0, #16
+; CHECK-GI-NEXT: fcmeq.4s v4, v0, v0
; CHECK-GI-NEXT: add.4s v2, v0, v2
+; CHECK-GI-NEXT: orr.16b v0, v0, v5
; CHECK-GI-NEXT: and.16b v1, v3, v1
-; CHECK-GI-NEXT: fcmeq.4s v3, v0, v0
-; CHECK-GI-NEXT: orr.4s v0, #64, lsl #16
-; CHECK-GI-NEXT: add.4s v1, v1, v2
-; CHECK-GI-NEXT: bit.16b v0, v1, v3
+; CHECK-GI-NEXT: mvn.16b v3, v4
+; CHECK-GI-NEXT: add.4s v1, v2, v1
+; CHECK-GI-NEXT: bif.16b v0, v1, v3
; CHECK-GI-NEXT: shrn.4h v0, v0, #16
; CHECK-GI-NEXT: ret
%vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat>
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index b3b9653b14481..710ca72c9b2fd 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -38,7 +38,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fptrunc_double
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_powi
@@ -62,7 +61,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_copysign_f64
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_floor
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
@@ -106,7 +104,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fptrunc_double
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_powi
@@ -130,7 +127,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_copysign_f64
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_floor
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
@@ -1117,24 +1113,40 @@ define bfloat @test_fptrunc_float(float %a) #0 {
}
define bfloat @test_fptrunc_double(double %a) #0 {
-; CHECK-CVT-LABEL: test_fptrunc_double:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: fcvtxn s0, d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fptrunc_double:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: fcvtxn s0, d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_fptrunc_double:
; CHECK-BF16: // %bb.0:
; CHECK-BF16-NEXT: fcvtxn s0, d0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fptrunc_double:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: fcvtxn s0, d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = fptrunc double %a to bfloat
ret bfloat %r
}
@@ -1994,28 +2006,57 @@ define bfloat @test_copysign_f32(bfloat %a, float %b) #0 {
}
define bfloat @test_copysign_f64(bfloat %a, double %b) #0 {
-; CHECK-CVT-LABEL: test_copysign_f64:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: fcvt s1, d1
-; CHECK-CVT-NEXT: mvni v2.4s, #128, lsl #24
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v2.16b
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_copysign_f64:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: fcvt s1, d1
+; CHECK-CVT-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_copysign_f64:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: fcvt s1, d1
-; CHECK-BF16-NEXT: mvni v2.4s, #128, lsl #24
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: bif v0.16b, v1.16b, v2.16b
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_copysign_f64:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: fcvt s1, d1
+; CHECK-BF16-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign_f64:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: fcvtxn s1, d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: fmov w9, s1
+; CHECK-CVT-GI-NEXT: fcmp s1, #0.0
+; CHECK-CVT-GI-NEXT: mvni v1.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s2, w8
+; CHECK-CVT-GI-NEXT: bif v0.8b, v2.8b, v1.8b
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_copysign_f64:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: fcvtxn s1, d1
+; CHECK-BF16-GI-NEXT: mvni v2.4h, #128, lsl #8
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: bfcvt h1, s1
+; CHECK-BF16-GI-NEXT: bif v0.8b, v1.8b, v2.8b
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
+; CHECK-BF16-GI-NEXT: ret
%tb = fptrunc double %b to bfloat
%r = call bfloat @llvm.copysign.bf16(bfloat %a, bfloat %tb)
ret bfloat %r
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index b292a3de4f9af..8402208837d38 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -8,7 +8,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sub_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for mul_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i32
@@ -40,7 +39,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sub_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for mul_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i32
@@ -260,21 +258,21 @@ define <4 x bfloat> @s_to_h(<4 x float> %a) {
}
define <4 x bfloat> @d_to_h(<4 x double> %a) {
-; CHECK-CVT-LABEL: d_to_h:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-NEXT: fcmeq v3.4s, v0.4s, v0.4s
-; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v3.16b
-; CHECK-CVT-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: d_to_h:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-SD-NEXT: fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT: bit v0.16b, v1.16b, v3.16b
+; CHECK-CVT-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: d_to_h:
; CHECK-BF16: // %bb.0:
@@ -282,6 +280,24 @@ define <4 x bfloat> @d_to_h(<4 x double> %a) {
; CHECK-BF16-NEXT: fcvtxn2 v0.4s, v1.2d
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: d_to_h:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc <4 x double> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 626d54acdb6b8..be95f9cc177ba 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -9,7 +9,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sub_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for mul_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v4i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v8i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v16i8
@@ -45,7 +44,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sub_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for mul_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v4i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v8i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v16i8
@@ -361,30 +359,30 @@ define <8 x bfloat> @s_to_h(<8 x float> %a) {
}
define <8 x bfloat> @d_to_h(<8 x double> %a) {
-; CHECK-CVT-LABEL: d_to_h:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-CVT-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v3.2d
-; CHECK-CVT-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-CVT-NEXT: add v6.4s, v2.4s, v3.4s
-; CHECK-CVT-NEXT: add v3.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: and v4.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s
-; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v4.4s, v4.4s, v6.4s
-; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-CVT-NEXT: bit v2.16b, v4.16b, v5.16b
-; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v6.16b
-; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: d_to_h:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v2.4s, v3.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: add v6.4s, v2.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-SD-NEXT: fcmeq v6.4s, v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: bit v2.16b, v4.16b, v5.16b
+; CHECK-CVT-SD-NEXT: bit v0.16b, v1.16b, v6.16b
+; CHECK-CVT-SD-NEXT: uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: d_to_h:
; CHECK-BF16: // %bb.0:
@@ -395,6 +393,36 @@ define <8 x bfloat> @d_to_h(<8 x double> %a) {
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v2.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: d_to_h:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v2.4s, v3.2d
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v2.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc <8 x double> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
More information about the llvm-commits
mailing list