[llvm] [GlobalISel] Improve bf16 converts with fast-math flags. (PR #200741)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 7 07:52:14 PDT 2026
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/200741
>From 1205172c53fb5846f85436fe0ab32f006d2561f1 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Mon, 1 Jun 2026 08:55:01 +0100
Subject: [PATCH 1/4] [GlobalISel] Improve bf16 converts with fast-math flags.
This alters the lowering of bf16 G_TRUNC to exclude the check for nan if the
operation being extended is nnan. Flags are then threaded through so that the
G_FPEXT and G_FPTRUNC from promoted nodes keep the same FMF.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 5 +-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 58 ++--
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 19 +-
llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll | 10 +-
.../test/CodeGen/AArch64/bf16-instructions.ll | 23 +-
.../CodeGen/AArch64/bf16-v4-instructions.ll | 269 ++++++++----------
.../CodeGen/AArch64/bf16-v8-instructions.ll | 241 ++++++----------
.../AMDGPU/GlobalISel/legalize-fdiv.mir | 6 +-
.../AMDGPU/GlobalISel/legalize-fmad.s16.mir | 102 ++++---
.../AMDGPU/GlobalISel/legalize-fpow.mir | 19 +-
.../AMDGPU/GlobalISel/legalize-fpowi.mir | 6 +-
11 files changed, 347 insertions(+), 411 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 8d6407a7c0115..bf86649102de8 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -146,7 +146,7 @@ class LegalizerHelper {
/// ExtOpcode for the extension instruction, and replacing the vreg of the
/// operand in place.
LLVM_ABI void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx,
- unsigned ExtOpcode);
+ unsigned ExtOpcode, unsigned Flags = 0);
/// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
/// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and
@@ -157,7 +157,8 @@ class LegalizerHelper {
/// Def by extending the operand's type to \p WideTy and truncating it back
/// with the \p TruncOpcode, and replacing the vreg of the operand in place.
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0,
- unsigned TruncOpcode = TargetOpcode::G_TRUNC);
+ unsigned TruncOpcode = TargetOpcode::G_TRUNC,
+ unsigned Flags = 0);
// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
// Def by truncating the operand's type to \p NarrowTy, replacing in place and
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 021f7233ce3b7..8ba10b4dadb02 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2161,9 +2161,10 @@ Register LegalizerHelper::coerceToScalar(Register Val) {
}
void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
- unsigned OpIdx, unsigned ExtOpcode) {
+ unsigned OpIdx, unsigned ExtOpcode,
+ unsigned Flags) {
MachineOperand &MO = MI.getOperand(OpIdx);
- auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
+ auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}, Flags);
MO.setReg(ExtB.getReg(0));
}
@@ -2175,11 +2176,12 @@ void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
}
void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
- unsigned OpIdx, unsigned TruncOpcode) {
+ unsigned OpIdx, unsigned TruncOpcode,
+ unsigned Flags) {
MachineOperand &MO = MI.getOperand(OpIdx);
Register DstExt = MRI.createGenericVirtualRegister(WideTy);
MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
- MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
+ MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}, Flags);
MO.setReg(DstExt);
}
@@ -3110,7 +3112,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return UnableToLegalize;
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
case TargetOpcode::G_FPTOSI:
@@ -3123,7 +3125,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
@@ -3131,7 +3133,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0)
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
else
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
@@ -3141,7 +3143,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0)
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
else
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
@@ -3184,7 +3186,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
}
MIRBuilder.buildTrunc(OldDst, NewDst);
} else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
@@ -3279,8 +3281,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else {
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
- widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT, MI.getFlags());
}
Observer.changedInstr(MI);
return Legalized;
@@ -3433,18 +3435,18 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
- widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
case TargetOpcode::G_FMODF: {
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarDst(MI, WideTy, 1, TargetOpcode::G_FPTRUNC);
+ widenScalarDst(MI, WideTy, 1, TargetOpcode::G_FPTRUNC, MI.getFlags());
MIRBuilder.setInsertPt(MIRBuilder.getMBB(), --MIRBuilder.getInsertPt());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
}
@@ -3456,8 +3458,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return UnableToLegalize;
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
}
@@ -3477,8 +3479,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0) {
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
} else {
widenScalarDst(MI, WideTy, 1);
}
@@ -3493,7 +3495,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
@@ -3570,8 +3572,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Register VecReg = MI.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
LLT WideVecTy = VecTy.changeElementType(WideTy);
- widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT);
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
Observer.changedInstr(MI);
return Legalized;
}
@@ -8885,8 +8887,6 @@ LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
LLT I16Ty = SrcTy.changeElementType(LLT::integer(16));
LLT I32Ty = SrcTy.changeElementType(LLT::integer(32));
- auto IsNaN = MIRBuilder.buildFCmp(CmpInst::FCMP_UNO, I1Ty, SrcReg,
- MIRBuilder.buildFConstant(SrcTy, 0));
auto SrcI = MIRBuilder.buildBitcast(I32Ty, SrcReg);
// Conversions should set NaN's quiet bit. This also prevents NaNs from
@@ -8904,11 +8904,15 @@ LegalizerHelper::lowerFPTRUNC_F32_TO_BF16(MachineInstr &MI) {
// Don't round if we had a NaN, we don't want to turn 0x7fffffff into
// 0x80000000.
- auto Sel = MIRBuilder.buildSelect(I32Ty, IsNaN, NaN, Add);
+ if (!MI.getFlag(MachineInstr::FmNoNans)) {
+ auto IsNaN = MIRBuilder.buildFCmp(CmpInst::FCMP_UNO, I1Ty, SrcReg,
+ MIRBuilder.buildFConstant(SrcTy, 0));
+ Add = MIRBuilder.buildSelect(I32Ty, IsNaN, NaN, Add);
+ }
// Now that we have rounded, shift the bits into position.
auto Srl =
- MIRBuilder.buildLShr(I32Ty, Sel, MIRBuilder.buildConstant(I32Ty, 16));
+ MIRBuilder.buildLShr(I32Ty, Add, MIRBuilder.buildConstant(I32Ty, 16));
auto Trunc = MIRBuilder.buildTrunc(I16Ty, Srl);
MIRBuilder.buildBitcast(DstReg, Trunc);
MI.eraseFromParent();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e4e58327ead53..fe05f4f2a6dcd 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -2686,9 +2686,9 @@ bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
// G_FPTRUNC.
if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
- auto Mid =
- MIRBuilder.buildInstr(AArch64::G_FPTRUNC_ODD, {LLT::float32()}, {Src});
- MIRBuilder.buildInstr(AArch64::G_FPTRUNC, {Dst}, {Mid});
+ auto Mid = MIRBuilder.buildInstr(AArch64::G_FPTRUNC_ODD, {LLT::float32()},
+ {Src}, MI.getFlags());
+ MIRBuilder.buildInstr(AArch64::G_FPTRUNC, {Dst}, {Mid}, MI.getFlags());
MI.eraseFromParent();
return true;
}
@@ -2726,9 +2726,10 @@ bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
// Create all of the round-to-odd instructions and store them
for (auto SrcReg : RegsToUnmergeTo) {
- Register Mid =
- MIRBuilder.buildInstr(AArch64::G_FPTRUNC_ODD, {v2s32}, {SrcReg})
- .getReg(0);
+ Register Mid = MIRBuilder
+ .buildInstr(AArch64::G_FPTRUNC_ODD, {v2s32}, {SrcReg},
+ MI.getFlags())
+ .getReg(0);
TruncOddDstRegs.push_back(Mid);
}
@@ -2744,10 +2745,12 @@ bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
.getReg(0);
RegsToMerge.push_back(
- MIRBuilder.buildFPTrunc(v4s16, ConcatDst).getReg(0));
+ MIRBuilder.buildFPTrunc(v4s16, ConcatDst, MI.getFlags()).getReg(0));
} else {
RegsToMerge.push_back(
- MIRBuilder.buildFPTrunc(v2s16, TruncOddDstRegs[Index++]).getReg(0));
+ MIRBuilder
+ .buildFPTrunc(v2s16, TruncOddDstRegs[Index++], MI.getFlags())
+ .getReg(0));
}
}
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
index b2d9237531900..43137f4b97feb 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -171,15 +171,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp {
; CHECK-GI-NEXT: fcvtxn v0.2s, v0.2d
; CHECK-GI-NEXT: movi.4s v1, #1
; CHECK-GI-NEXT: movi.4s v2, #127, msl #8
-; CHECK-GI-NEXT: movi.4s v5, #64, lsl #16
+; CHECK-GI-NEXT: movi.4s v4, #64, lsl #16
; CHECK-GI-NEXT: ushr.4s v3, v0, #16
-; CHECK-GI-NEXT: fcmeq.4s v4, v0, v0
+; CHECK-GI-NEXT: fcmeq.4s v5, v0, v0
; CHECK-GI-NEXT: add.4s v2, v0, v2
-; CHECK-GI-NEXT: orr.16b v0, v0, v5
+; CHECK-GI-NEXT: orr.16b v0, v0, v4
; CHECK-GI-NEXT: and.16b v1, v3, v1
-; CHECK-GI-NEXT: mvn.16b v3, v4
; CHECK-GI-NEXT: add.4s v1, v2, v1
-; CHECK-GI-NEXT: bif.16b v0, v1, v3
+; CHECK-GI-NEXT: mvn.16b v2, v5
+; CHECK-GI-NEXT: bif.16b v0, v1, v2
; CHECK-GI-NEXT: shrn.4h v0, v0, #16
; CHECK-GI-NEXT: ret
%vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat>
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index 2ddfdeed09f85..8790347209776 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -129,12 +129,9 @@ define bfloat @test_fadd_fast(bfloat %a, bfloat %b) #0 {
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
; CHECK-CVT-GI-NEXT: fmov w9, s0
-; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
; CHECK-CVT-GI-NEXT: add w8, w9, w8
-; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
; CHECK-CVT-GI-NEXT: add w8, w8, w10
-; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
; CHECK-CVT-GI-NEXT: fmov s0, w8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
@@ -326,23 +323,17 @@ define bfloat @test_fmadd(bfloat %a, bfloat %b, bfloat %c) #0 {
; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
; CHECK-CVT-GI-NEXT: shll v1.4s, v2.4h, #16
; CHECK-CVT-GI-NEXT: fmov w8, s0
-; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-GI-NEXT: add w11, w8, w10
-; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
-; CHECK-CVT-GI-NEXT: add w9, w11, w9
-; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: add w8, w8, w9
; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
; CHECK-CVT-GI-NEXT: fmov s0, w8
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
; CHECK-CVT-GI-NEXT: fmov w8, s0
-; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-GI-NEXT: add w10, w8, w10
-; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
-; CHECK-CVT-GI-NEXT: add w9, w10, w9
-; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: add w8, w8, w9
; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
; CHECK-CVT-GI-NEXT: fmov s0, w8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
@@ -1924,12 +1915,9 @@ define bfloat @test_fptrunc_float_fast(float %a) #0 {
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: fmov w9, s0
; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
; CHECK-CVT-GI-NEXT: add w8, w9, w8
-; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
; CHECK-CVT-GI-NEXT: add w8, w8, w10
-; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
; CHECK-CVT-GI-NEXT: fmov s0, w8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
@@ -1963,12 +1951,9 @@ define bfloat @test_fptrunc_double_fast(double %a) #0 {
; CHECK-CVT-GI-NEXT: fcvtxn s0, d0
; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
; CHECK-CVT-GI-NEXT: fmov w9, s0
-; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
; CHECK-CVT-GI-NEXT: add w8, w9, w8
-; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
; CHECK-CVT-GI-NEXT: add w8, w8, w10
-; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
; CHECK-CVT-GI-NEXT: fmov s0, w8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index 6c6a9714f7deb..233092aae8e28 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -93,17 +93,17 @@ define <4 x bfloat> @test_fadd(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -144,19 +144,13 @@ define <4 x bfloat> @test_fadd_fast(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
-; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
-; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-BF16-GI-LABEL: test_fadd_fast:
@@ -197,17 +191,17 @@ define <4 x bfloat> @test_fsub(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -249,17 +243,17 @@ define <4 x bfloat> @test_fmul(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -311,31 +305,20 @@ define <4 x bfloat> @test_fmadd(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v7.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v5.16b
-; CHECK-CVT-GI-NEXT: add v4.4s, v7.4s, v4.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: and v3.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v3.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v3.4s
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
-; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-BF16-GI-LABEL: test_fmadd:
@@ -381,17 +364,17 @@ define <4 x bfloat> @test_fdiv(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -1536,14 +1519,14 @@ define <4 x bfloat> @test_fptrunc_float(<4 x float> %a) {
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc <4 x float> %a to <4 x bfloat>
@@ -1578,17 +1561,17 @@ define <4 x bfloat> @test_fptrunc_double(<4 x double> %a) {
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc <4 x double> %a to <4 x bfloat>
@@ -1618,17 +1601,11 @@ define <4 x bfloat> @test_fptrunc_float_fast(<4 x float> %a) {
; CHECK-CVT-GI-LABEL: test_fptrunc_float_fast:
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
-; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
-; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc fast <4 x float> %a to <4 x bfloat>
ret <4 x bfloat> %1
@@ -1661,19 +1638,13 @@ define <4 x bfloat> @test_fptrunc_double_fast(<4 x double> %a) {
; CHECK-CVT-GI-LABEL: test_fptrunc_double_fast:
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
-; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
-; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc fast <4 x double> %a to <4 x bfloat>
ret <4 x bfloat> %1
@@ -1872,16 +1843,16 @@ define <4 x bfloat> @test_sqrt(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fsqrt v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -5787,18 +5758,18 @@ define <4 x bfloat> @test_fma(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
; CHECK-CVT-GI-NEXT: movi v0.4s, #1
; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: ushr v3.4s, v2.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v2.4s, v2.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-GI-NEXT: bit v0.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v1.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v0.16b, v2.16b, v1.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -5941,17 +5912,17 @@ define <4 x bfloat> @test_minnum(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fminnm v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -6057,17 +6028,17 @@ define <4 x bfloat> @test_maxnum(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: fmaxnm v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
@@ -6121,14 +6092,14 @@ define <4 x bfloat> @test_copysign_f32(<4 x bfloat> %a, <4 x float> %b) #0 {
; CHECK-CVT-GI-NEXT: movi v2.4s, #1
; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fcmeq v6.4s, v1.4s, v1.4s
; CHECK-CVT-GI-NEXT: and v2.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: add v3.4s, v1.4s, v3.4s
-; CHECK-CVT-GI-NEXT: mvn v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v5.16b
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
-; CHECK-CVT-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v6.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v2.16b, v3.16b
; CHECK-CVT-GI-NEXT: mvni v2.4h, #128, lsl #8
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: bif v0.8b, v1.8b, v2.8b
@@ -6163,16 +6134,16 @@ define <4 x bfloat> @test_floor(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frintm v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.floor.v4bf16(<4 x bfloat> %a)
@@ -6204,16 +6175,16 @@ define <4 x bfloat> @test_ceil(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frintp v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.ceil.v4bf16(<4 x bfloat> %a)
@@ -6245,16 +6216,16 @@ define <4 x bfloat> @test_trunc(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frintz v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.trunc.v4bf16(<4 x bfloat> %a)
@@ -6286,16 +6257,16 @@ define <4 x bfloat> @test_rint(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frintx v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.rint.v4bf16(<4 x bfloat> %a)
@@ -6327,16 +6298,16 @@ define <4 x bfloat> @test_nearbyint(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frinti v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.nearbyint.v4bf16(<4 x bfloat> %a)
@@ -6368,16 +6339,16 @@ define <4 x bfloat> @test_round(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frinta v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.round.v4bf16(<4 x bfloat> %a)
@@ -6409,16 +6380,16 @@ define <4 x bfloat> @test_roundeven(<4 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: frintn v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.roundeven.v4bf16(<4 x bfloat> %a)
@@ -6463,18 +6434,18 @@ define <4 x bfloat> @test_fmuladd(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v6.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: add v7.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v5.16b
+; CHECK-CVT-GI-NEXT: mvn v6.16b, v6.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v7.4s, v4.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v6.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
@@ -6482,10 +6453,10 @@ define <4 x bfloat> @test_fmuladd(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 971d9e2a6e2b5..865249b5e5880 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -148,11 +148,11 @@ define <8 x bfloat> @test_fadd(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -220,33 +220,22 @@ define <8 x bfloat> @test_fadd_fast(<8 x bfloat> %a, <8 x bfloat> %b) {
;
; CHECK-CVT-GI-LABEL: test_fadd_fast:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v0.4h, #16
; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll v4.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
-; CHECK-CVT-GI-NEXT: fadd v2.4s, v2.4s, v3.4s
-; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
-; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
-; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
-; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
-; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
-; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fadd v3.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: add v0.4s, v0.4s, v5.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v2.4s, v5.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v3.4s, v0.4s
+; CHECK-CVT-GI-NEXT: addhn v1.4h, v1.4s, v2.4s
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-CVT-GI-NEXT: ret
;
@@ -332,11 +321,11 @@ define <8 x bfloat> @test_fsub(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -424,11 +413,11 @@ define <8 x bfloat> @test_fmul(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -529,53 +518,32 @@ define <8 x bfloat> @test_fmadd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %
; CHECK-CVT-GI-NEXT: shll v4.4s, v1.4h, #16
; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-GI-NEXT: movi v7.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: fmul v3.4s, v3.4s, v4.4s
-; CHECK-CVT-GI-NEXT: movi v4.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v16.4s, v3.4s, v3.4s
-; CHECK-CVT-GI-NEXT: add v17.4s, v3.4s, v4.4s
-; CHECK-CVT-GI-NEXT: ushr v6.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v18.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v19.4s, v0.4s, v4.4s
-; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-CVT-GI-NEXT: and v6.16b, v6.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v16.16b, v16.16b
-; CHECK-CVT-GI-NEXT: add v5.4s, v17.4s, v5.4s
-; CHECK-CVT-GI-NEXT: mvn v17.16b, v18.16b
-; CHECK-CVT-GI-NEXT: add v6.4s, v19.4s, v6.4s
-; CHECK-CVT-GI-NEXT: bif v3.16b, v5.16b, v16.16b
-; CHECK-CVT-GI-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: add v4.4s, v4.4s, v6.4s
+; CHECK-CVT-GI-NEXT: add v5.4s, v5.4s, v6.4s
+; CHECK-CVT-GI-NEXT: addhn v3.4h, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: shll v4.4s, v2.4h, #16
; CHECK-CVT-GI-NEXT: shll2 v2.4s, v2.8h, #16
-; CHECK-CVT-GI-NEXT: bif v0.16b, v6.16b, v17.16b
-; CHECK-CVT-GI-NEXT: shrn v3.4h, v3.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v5.4s
; CHECK-CVT-GI-NEXT: shll v3.4s, v3.4h, #16
; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-GI-NEXT: fadd v3.4s, v3.4s, v5.4s
-; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
-; CHECK-CVT-GI-NEXT: ushr v2.4s, v3.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v6.4s, v3.4s, v3.4s
-; CHECK-CVT-GI-NEXT: add v16.4s, v3.4s, v4.4s
-; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: add v4.4s, v0.4s, v4.4s
-; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
-; CHECK-CVT-GI-NEXT: and v2.16b, v2.16b, v1.16b
-; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v6.16b
-; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
-; CHECK-CVT-GI-NEXT: add v2.4s, v16.4s, v2.4s
-; CHECK-CVT-GI-NEXT: add v1.4s, v4.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bit v2.16b, v3.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: fadd v3.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: fadd v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v0.4s, v0.4s, v6.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v3.4s, v0.4s
+; CHECK-CVT-GI-NEXT: addhn v1.4h, v2.4s, v1.4s
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-CVT-GI-NEXT: ret
;
@@ -671,10 +639,10 @@ define <8 x bfloat> @test_fdiv(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -3051,12 +3019,12 @@ define <8 x bfloat> @test_fptrunc_float(<8 x float> %a) {
; CHECK-CVT-GI-NEXT: add v3.4s, v1.4s, v3.4s
; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v16.16b
; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v17.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v16.16b
; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bif v1.16b, v2.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v2.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -3121,11 +3089,11 @@ define <8 x bfloat> @test_fptrunc_double(<8 x double> %a) {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v2.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v2.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -3165,26 +3133,15 @@ define <8 x bfloat> @test_fptrunc_float_fast(<8 x float> %a) {
; CHECK-CVT-GI-LABEL: test_fptrunc_float_fast:
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: movi v2.4s, #1
-; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: ushr v5.4s, v1.4s, #16
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
-; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: fcmeq v16.4s, v1.4s, v1.4s
-; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
-; CHECK-CVT-GI-NEXT: add v17.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
-; CHECK-CVT-GI-NEXT: add v3.4s, v1.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v16.16b
-; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
-; CHECK-CVT-GI-NEXT: add v4.4s, v17.4s, v4.4s
-; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bif v1.16b, v2.16b, v7.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: and v3.16b, v3.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: add v3.4s, v3.4s, v5.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v2.4s, v5.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: addhn v1.4h, v1.4s, v2.4s
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc fast <8 x float> %a to <8 x bfloat>
@@ -3231,29 +3188,18 @@ define <8 x bfloat> @test_fptrunc_double_fast(<8 x double> %a) {
; CHECK-CVT-GI: // %bb.0:
; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
; CHECK-CVT-GI-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: movi v5.4s, #127, msl #8
; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
; CHECK-CVT-GI-NEXT: fcvtxn2 v2.4s, v3.2d
; CHECK-CVT-GI-NEXT: movi v1.4s, #1
-; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
-; CHECK-CVT-GI-NEXT: ushr v5.4s, v2.4s, #16
-; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v0.4s, v0.4s
-; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v2.4s, v2.4s
-; CHECK-CVT-GI-NEXT: add v16.4s, v0.4s, v3.4s
-; CHECK-CVT-GI-NEXT: add v3.4s, v2.4s, v3.4s
-; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
-; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
-; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
-; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
-; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
-; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v2.16b, v7.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: and v3.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v3.4s, v3.4s, v5.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-CVT-GI-NEXT: addhn v0.4h, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: addhn v1.4h, v2.4s, v1.4s
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-CVT-GI-NEXT: ret
%1 = fptrunc fast <8 x double> %a to <8 x bfloat>
@@ -3665,10 +3611,10 @@ define <8 x bfloat> @test_sqrt(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -12345,8 +12291,8 @@ define <8 x bfloat> @test_fma(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c)
; CHECK-CVT-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
; CHECK-CVT-GI-NEXT: movi v0.4s, #1
; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
-; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v5.4s, v5.4s
; CHECK-CVT-GI-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v5.4s, v5.4s
; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v2.4s, v2.4s
; CHECK-CVT-GI-NEXT: add v16.4s, v5.4s, v1.4s
@@ -12356,14 +12302,13 @@ define <8 x bfloat> @test_fma(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c)
; CHECK-CVT-GI-NEXT: and v0.16b, v4.16b, v0.16b
; CHECK-CVT-GI-NEXT: orr v4.16b, v5.16b, v6.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v3.4s, v16.4s, v3.4s
; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-GI-NEXT: mov v1.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bsl v1.16b, v4.16b, v3.16b
-; CHECK-CVT-GI-NEXT: bif v2.16b, v0.16b, v7.16b
-; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
-; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mvn v1.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bit v3.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bsl v1.16b, v2.16b, v0.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-CVT-GI-NEXT: ret
;
@@ -12680,11 +12625,11 @@ define <8 x bfloat> @test_minnum(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -12943,11 +12888,11 @@ define <8 x bfloat> @test_maxnum(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13025,12 +12970,12 @@ define <8 x bfloat> @test_copysign_f32(<8 x bfloat> %a, <8 x float> %b) #0 {
; CHECK-CVT-GI-NEXT: add v4.4s, v2.4s, v4.4s
; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v7.16b
; CHECK-CVT-GI-NEXT: mvn v6.16b, v16.16b
-; CHECK-CVT-GI-NEXT: mvn v16.16b, v17.16b
; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v7.16b
; CHECK-CVT-GI-NEXT: add v5.4s, v18.4s, v5.4s
; CHECK-CVT-GI-NEXT: add v3.4s, v4.4s, v3.4s
+; CHECK-CVT-GI-NEXT: mvn v4.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v5.16b, v6.16b
-; CHECK-CVT-GI-NEXT: bif v2.16b, v3.16b, v16.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v3.16b, v4.16b
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v2.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v1.d[1], v2.d[0]
@@ -13101,11 +13046,11 @@ define <8 x bfloat> @test_floor(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13183,11 +13128,11 @@ define <8 x bfloat> @test_ceil(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13265,11 +13210,11 @@ define <8 x bfloat> @test_trunc(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13347,11 +13292,11 @@ define <8 x bfloat> @test_rint(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13429,11 +13374,11 @@ define <8 x bfloat> @test_nearbyint(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13511,11 +13456,11 @@ define <8 x bfloat> @test_round(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13593,11 +13538,11 @@ define <8 x bfloat> @test_roundeven(<8 x bfloat> %a) #0 {
; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
-; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
@@ -13736,11 +13681,11 @@ define <8 x bfloat> @test_fmuladd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat>
; CHECK-CVT-GI-NEXT: and v2.16b, v2.16b, v1.16b
; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-CVT-GI-NEXT: mvn v5.16b, v6.16b
-; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
; CHECK-CVT-GI-NEXT: add v2.4s, v16.4s, v2.4s
; CHECK-CVT-GI-NEXT: add v1.4s, v4.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v4.16b, v17.16b
; CHECK-CVT-GI-NEXT: bit v2.16b, v3.16b, v5.16b
-; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v4.16b
; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
index 3ace984c6e98a..51635b94f5345 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
@@ -177,11 +177,11 @@ body: |
; SI-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
- ; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+ ; SI-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = afn G_FPEXT [[TRUNC]](s16)
+ ; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = afn G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[INT:%[0-9]+]]:_(s32) = afn G_INTRINSIC intrinsic(@llvm.amdgcn.rcp), [[FPEXT1]](s32)
; SI-NEXT: [[FMUL:%[0-9]+]]:_(s32) = afn G_FMUL [[FPEXT]], [[INT]]
- ; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+ ; SI-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = afn G_FPTRUNC [[FMUL]](s32)
; SI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
; SI-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
index ad472f95dc313..073a089a2721a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
@@ -34,6 +34,7 @@ body: |
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX7-LABEL: name: test_fmad_s16_flush
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -46,6 +47,7 @@ body: |
; GFX7-NEXT: [[FMAD:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC]], [[TRUNC1]], [[TRUNC2]]
; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FMAD]](s16)
; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX10-LABEL: name: test_fmad_s16_flush
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -122,6 +124,7 @@ body: |
; GFX6-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v2s16_flush
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -149,6 +152,7 @@ body: |
; GFX7-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v2s16_flush
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -270,6 +274,7 @@ body: |
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v4s16_flush
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX7-NEXT: {{ $}}
@@ -320,6 +325,7 @@ body: |
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v4s16_flush
; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX10-NEXT: {{ $}}
@@ -404,6 +410,7 @@ body: |
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX7-LABEL: name: test_fmad_s16_denorm
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -417,6 +424,7 @@ body: |
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC2]]
; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX10-LABEL: name: test_fmad_s16_denorm
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -461,16 +469,17 @@ body: |
; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
; GFX6-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
- ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+ ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT]], [[FPEXT1]]
- ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
- ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
+ ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL]](s32)
+ ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT2]], [[FPEXT3]]
- ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+ ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD]](s32)
; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX7-LABEL: name: test_fmad_s16_denorm_flags
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -484,6 +493,7 @@ body: |
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC2]]
; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX10-LABEL: name: test_fmad_s16_denorm_flags
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -560,6 +570,7 @@ body: |
; GFX6-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v2s16_denorm
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -589,6 +600,7 @@ body: |
; GFX7-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v2s16_denorm
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -651,28 +663,29 @@ body: |
; GFX6-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
; GFX6-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
; GFX6-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
- ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
+ ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT]], [[FPEXT1]]
- ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
- ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
+ ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL]](s32)
+ ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC4]](s16)
; GFX6-NEXT: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT2]], [[FPEXT3]]
- ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
- ; GFX6-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
- ; GFX6-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
+ ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD]](s32)
+ ; GFX6-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
+ ; GFX6-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC3]](s16)
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT4]], [[FPEXT5]]
- ; GFX6-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
- ; GFX6-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC2]](s16)
- ; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
+ ; GFX6-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL1]](s32)
+ ; GFX6-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC2]](s16)
+ ; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC5]](s16)
; GFX6-NEXT: [[FADD1:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT6]], [[FPEXT7]]
- ; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+ ; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v2s16_denorm_flags
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX7-NEXT: {{ $}}
@@ -702,6 +715,7 @@ body: |
; GFX7-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v2s16_denorm_flags
; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
@@ -823,6 +837,7 @@ body: |
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v4s16_denorm
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX7-NEXT: {{ $}}
@@ -877,6 +892,7 @@ body: |
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v4s16_denorm
; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX10-NEXT: {{ $}}
@@ -976,38 +992,38 @@ body: |
; GFX6-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
; GFX6-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
- ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
+ ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC4]](s16)
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT]], [[FPEXT1]]
- ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
- ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC8]](s16)
+ ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL]](s32)
+ ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC8]](s16)
; GFX6-NEXT: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT2]], [[FPEXT3]]
- ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
- ; GFX6-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
- ; GFX6-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
+ ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD]](s32)
+ ; GFX6-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
+ ; GFX6-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC5]](s16)
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT4]], [[FPEXT5]]
- ; GFX6-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
- ; GFX6-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC2]](s16)
- ; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC9]](s16)
+ ; GFX6-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL1]](s32)
+ ; GFX6-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC2]](s16)
+ ; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC9]](s16)
; GFX6-NEXT: [[FADD1:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT6]], [[FPEXT7]]
- ; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; GFX6-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
- ; GFX6-NEXT: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC6]](s16)
+ ; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD1]](s32)
+ ; GFX6-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC2]](s16)
+ ; GFX6-NEXT: [[FPEXT9:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC6]](s16)
; GFX6-NEXT: [[FMUL2:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT8]], [[FPEXT9]]
- ; GFX6-NEXT: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL2]](s32)
- ; GFX6-NEXT: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC4]](s16)
- ; GFX6-NEXT: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC10]](s16)
+ ; GFX6-NEXT: [[FPTRUNC4:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL2]](s32)
+ ; GFX6-NEXT: [[FPEXT10:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC4]](s16)
+ ; GFX6-NEXT: [[FPEXT11:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC10]](s16)
; GFX6-NEXT: [[FADD2:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT10]], [[FPEXT11]]
- ; GFX6-NEXT: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
- ; GFX6-NEXT: [[FPEXT12:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
- ; GFX6-NEXT: [[FPEXT13:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
+ ; GFX6-NEXT: [[FPTRUNC5:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD2]](s32)
+ ; GFX6-NEXT: [[FPEXT12:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC3]](s16)
+ ; GFX6-NEXT: [[FPEXT13:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC7]](s16)
; GFX6-NEXT: [[FMUL3:%[0-9]+]]:_(s32) = nnan G_FMUL [[FPEXT12]], [[FPEXT13]]
- ; GFX6-NEXT: [[FPTRUNC6:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL3]](s32)
- ; GFX6-NEXT: [[FPEXT14:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC6]](s16)
- ; GFX6-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC11]](s16)
+ ; GFX6-NEXT: [[FPTRUNC6:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL3]](s32)
+ ; GFX6-NEXT: [[FPEXT14:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FPTRUNC6]](s16)
+ ; GFX6-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC11]](s16)
; GFX6-NEXT: [[FADD3:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT14]], [[FPEXT15]]
- ; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
+ ; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD3]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
@@ -1020,6 +1036,7 @@ body: |
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX7-LABEL: name: test_fmad_v4s16_denorm_flags
; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX7-NEXT: {{ $}}
@@ -1074,6 +1091,7 @@ body: |
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+ ;
; GFX10-LABEL: name: test_fmad_v4s16_denorm_flags
; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
; GFX10-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
index b6e5e0ac8330d..b646a1d5cc292 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
@@ -37,6 +37,7 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[FCMP1]](s1), [[C7]], [[C2]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[INT2]], [[SELECT3]]
; GFX6-NEXT: $vgpr0 = COPY [[FMUL1]](s32)
+ ;
; GFX9-LABEL: name: test_fpow_s32
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -119,6 +120,7 @@ body: |
; GFX6-NEXT: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INT5]], [[SELECT7]]
; GFX6-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMUL1]](s32), [[FMUL3]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+ ;
; GFX9-LABEL: name: test_fpow_v2s32
; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
; GFX9-NEXT: {{ $}}
@@ -230,6 +232,7 @@ body: |
; GFX6-NEXT: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[INT8]], [[SELECT11]]
; GFX6-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMUL1]](s32), [[FMUL3]](s32), [[FMUL5]](s32)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+ ;
; GFX9-LABEL: name: test_fpow_v3s32
; GFX9: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
; GFX9-NEXT: {{ $}}
@@ -325,6 +328,7 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = nnan nsz G_SELECT [[FCMP1]](s1), [[C7]], [[C2]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan nsz G_FMUL [[INT2]], [[SELECT3]]
; GFX6-NEXT: $vgpr0 = COPY [[FMUL1]](s32)
+ ;
; GFX9-LABEL: name: test_fpow_s32_flags
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -389,6 +393,7 @@ body: |
; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
; GFX6-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX9-LABEL: name: test_fpow_s16
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -466,6 +471,7 @@ body: |
; GFX6-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
+ ;
; GFX9-LABEL: name: test_fpow_v2s16
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -520,8 +526,8 @@ body: |
; GFX6-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX6-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
- ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
+ ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC2]](s16)
; GFX6-NEXT: [[INT:%[0-9]+]]:_(s32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](s32)
; GFX6-NEXT: [[INT1:%[0-9]+]]:_(s32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[INT]](s32), [[FPEXT1]](s32)
; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float -1.260000e+02
@@ -535,9 +541,9 @@ body: |
; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = nnan nsz G_SELECT [[FCMP]](s1), [[C4]], [[C5]]
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan nsz G_FMUL [[INT2]], [[SELECT1]]
- ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
- ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
- ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
+ ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = nnan nsz G_FPTRUNC [[FMUL]](s32)
+ ; GFX6-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC1]](s16)
+ ; GFX6-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = nnan nsz G_FPEXT [[TRUNC3]](s16)
; GFX6-NEXT: [[INT3:%[0-9]+]]:_(s32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT2]](s32)
; GFX6-NEXT: [[INT4:%[0-9]+]]:_(s32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[INT3]](s32), [[FPEXT3]](s32)
; GFX6-NEXT: [[FCMP1:%[0-9]+]]:_(s1) = nnan nsz G_FCMP floatpred(olt), [[INT4]](s32), [[C1]]
@@ -546,13 +552,14 @@ body: |
; GFX6-NEXT: [[INT5:%[0-9]+]]:_(s32) = nnan nsz G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[FADD1]](s32)
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = nnan nsz G_SELECT [[FCMP1]](s1), [[C4]], [[C5]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan nsz G_FMUL [[INT5]], [[SELECT3]]
- ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
+ ; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan nsz G_FPTRUNC [[FMUL1]](s32)
; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
+ ;
; GFX9-LABEL: name: test_fpow_v2s16_flags
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir
index 249350b41d4a1..0467803161735 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir
@@ -16,7 +16,7 @@ body: |
; GFX6-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; GFX6-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX6-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
- ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+ ; GFX6-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
; GFX6-NEXT: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY1]](s32)
; GFX6-NEXT: [[INT:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT]](s32)
; GFX6-NEXT: [[INT1:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[INT]](s32), [[SITOFP]](s32)
@@ -31,9 +31,10 @@ body: |
; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = nnan G_SELECT [[FCMP]](s1), [[C3]], [[C4]]
; GFX6-NEXT: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[INT2]], [[SELECT1]]
- ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+ ; GFX6-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FMUL]](s32)
; GFX6-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
; GFX6-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+ ;
; GFX9-LABEL: name: test_fpowi_s16_s32_flags
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
@@ -92,6 +93,7 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = nnan G_SELECT [[FCMP1]](s1), [[C7]], [[C2]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan G_FMUL [[INT2]], [[SELECT3]]
; GFX6-NEXT: $vgpr0 = COPY [[FMUL1]](s32)
+ ;
; GFX9-LABEL: name: test_fpowi_s32_s32_flags
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
>From e062771fa06bbee7ab551dcdbc32a5298a4fd45d Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 7 Jul 2026 08:51:34 +0100
Subject: [PATCH 2/4] Switch to using widenScalarSrcUsingFPExt and
widenScalarDstUsingFPTrunc
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 19 ++++-
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 75 ++++++++++++-------
2 files changed, 66 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index bf86649102de8..36977f0ddaa84 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -146,7 +146,14 @@ class LegalizerHelper {
/// ExtOpcode for the extension instruction, and replacing the vreg of the
/// operand in place.
LLVM_ABI void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx,
- unsigned ExtOpcode, unsigned Flags = 0);
+ unsigned ExtOpcode);
+
+ /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+ /// Use by extending the operand's type to \p WideTy using the G_FPEXT for the
+ /// extension instruction, and replacing the vreg of the operand in place.
+ /// Flags are copied from MI to the new extend.
+ LLVM_ABI void widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
+ unsigned OpIdx);
/// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
/// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and
@@ -157,8 +164,14 @@ class LegalizerHelper {
/// Def by extending the operand's type to \p WideTy and truncating it back
/// with the \p TruncOpcode, and replacing the vreg of the operand in place.
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0,
- unsigned TruncOpcode = TargetOpcode::G_TRUNC,
- unsigned Flags = 0);
+ unsigned TruncOpcode = TargetOpcode::G_TRUNC);
+
+ /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+ /// Def by extending the operand's type to \p WideTy and truncating it back
+ /// with G_FPTRUNC, and replacing the vreg of the operand in place. Flags are
+ /// copied from MI to the new trunc.
+ LLVM_ABI void widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
+ unsigned OpIdx = 0);
// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
// Def by truncating the operand's type to \p NarrowTy, replacing in place and
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 8ba10b4dadb02..9bf4be4707099 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2160,11 +2160,27 @@ Register LegalizerHelper::coerceToScalar(Register Val) {
return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
}
+// Extract just the fast-math flags from Flags
+static uint32_t extractFMF(uint32_t Flags) {
+ return Flags &
+ (MachineInstr::MIFlag::FmNoNans | MachineInstr::MIFlag::FmNoInfs |
+ MachineInstr::MIFlag::FmNsz | MachineInstr::MIFlag::FmArcp |
+ MachineInstr::MIFlag::FmContract | MachineInstr::MIFlag::FmAfn |
+ MachineInstr::MIFlag::FmReassoc);
+}
+
void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
- unsigned OpIdx, unsigned ExtOpcode,
- unsigned Flags) {
+ unsigned OpIdx, unsigned ExtOpcode) {
+ MachineOperand &MO = MI.getOperand(OpIdx);
+ auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
+ MO.setReg(ExtB.getReg(0));
+}
+
+void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
+ unsigned OpIdx) {
MachineOperand &MO = MI.getOperand(OpIdx);
- auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}, Flags);
+ auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, {WideTy}, {MO},
+ extractFMF(MI.getFlags()));
MO.setReg(ExtB.getReg(0));
}
@@ -2176,12 +2192,21 @@ void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
}
void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
- unsigned OpIdx, unsigned TruncOpcode,
- unsigned Flags) {
+ unsigned OpIdx, unsigned TruncOpcode) {
+ MachineOperand &MO = MI.getOperand(OpIdx);
+ Register DstExt = MRI.createGenericVirtualRegister(WideTy);
+ MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
+ MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
+ MO.setReg(DstExt);
+}
+
+void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
+ unsigned OpIdx) {
MachineOperand &MO = MI.getOperand(OpIdx);
Register DstExt = MRI.createGenericVirtualRegister(WideTy);
MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
- MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}, Flags);
+ MIRBuilder.buildInstr(TargetOpcode::G_FPTRUNC, {MO}, {DstExt},
+ extractFMF(MI.getFlags()));
MO.setReg(DstExt);
}
@@ -3112,7 +3137,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return UnableToLegalize;
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 1);
Observer.changedInstr(MI);
return Legalized;
case TargetOpcode::G_FPTOSI:
@@ -3125,7 +3150,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 1);
Observer.changedInstr(MI);
return Legalized;
@@ -3133,7 +3158,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0)
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
else
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
@@ -3143,7 +3168,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0)
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
else
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
@@ -3186,7 +3211,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
}
MIRBuilder.buildTrunc(OldDst, NewDst);
} else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 1);
Observer.changedInstr(MI);
return Legalized;
@@ -3281,8 +3306,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else {
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 2);
+ widenScalarSrcUsingFPExt(MI, WideTy, 3);
}
Observer.changedInstr(MI);
return Legalized;
@@ -3435,18 +3460,18 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
- widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, I);
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
Observer.changedInstr(MI);
return Legalized;
case TargetOpcode::G_FMODF: {
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 2);
- widenScalarDst(MI, WideTy, 1, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarDstUsingFPTrunc(MI, WideTy, 1);
MIRBuilder.setInsertPt(MIRBuilder.getMBB(), --MIRBuilder.getInsertPt());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
Observer.changedInstr(MI);
return Legalized;
}
@@ -3458,8 +3483,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return UnableToLegalize;
Observer.changingInstr(MI);
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 1);
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
Observer.changedInstr(MI);
return Legalized;
}
@@ -3479,8 +3504,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changingInstr(MI);
if (TypeIdx == 0) {
- widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 2);
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
} else {
widenScalarDst(MI, WideTy, 1);
}
@@ -3495,7 +3520,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx == 0)
widenScalarDst(MI, WideTy);
else
- widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideTy, 1);
Observer.changedInstr(MI);
return Legalized;
@@ -3572,8 +3597,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Register VecReg = MI.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
LLT WideVecTy = VecTy.changeElementType(WideTy);
- widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT, MI.getFlags());
- widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC, MI.getFlags());
+ widenScalarSrcUsingFPExt(MI, WideVecTy, 1);
+ widenScalarDstUsingFPTrunc(MI, WideTy, 0);
Observer.changedInstr(MI);
return Legalized;
}
>From 6e34b00ca5708fb671a076bd598ab9df4245aaaf Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 7 Jul 2026 08:55:24 +0100
Subject: [PATCH 3/4] rebase and update ldexp.ll
---
llvm/test/CodeGen/AArch64/ldexp.ll | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/ldexp.ll b/llvm/test/CodeGen/AArch64/ldexp.ll
index a849a398eddb1..fed1489b1bcff 100644
--- a/llvm/test/CodeGen/AArch64/ldexp.ll
+++ b/llvm/test/CodeGen/AArch64/ldexp.ll
@@ -4,6 +4,7 @@
; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+sve < %s -o - | FileCheck -check-prefixes=SVE,SVEWINDOWS %s
; RUN: llc -mtriple=aarch64-windows-msvc < %s -o - | FileCheck -check-prefixes=WINDOWS %s
+; GISEL: warning: Instruction selection used fallback path for testExpIntrinsic_i16
define double @testExp(double %val, i32 %a) {
; SVE-LABEL: testExp:
@@ -323,12 +324,9 @@ define bfloat @testExpbf16(bfloat %val, i32 %a) {
; GISEL-NEXT: bl ldexpf
; GISEL-NEXT: fmov w9, s0
; GISEL-NEXT: mov w8, #32767 // =0x7fff
-; GISEL-NEXT: fcmp s0, #0.0
; GISEL-NEXT: ubfx w10, w9, #16, #1
; GISEL-NEXT: add w8, w9, w8
-; GISEL-NEXT: orr w9, w9, #0x400000
; GISEL-NEXT: add w8, w8, w10
-; GISEL-NEXT: csel w8, w9, w8, vs
; GISEL-NEXT: lsr w8, w8, #16
; GISEL-NEXT: fmov s0, w8
; GISEL-NEXT: // kill: def $h0 killed $h0 killed $s0
>From 182631ca3e145593a8497b3924e1a420f4d300ab Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 7 Jul 2026 15:51:59 +0100
Subject: [PATCH 4/4] Drop extractFMF
---
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 14 ++------------
1 file changed, 2 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 9bf4be4707099..41630c70ebeb6 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2160,15 +2160,6 @@ Register LegalizerHelper::coerceToScalar(Register Val) {
return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
}
-// Extract just the fast-math flags from Flags
-static uint32_t extractFMF(uint32_t Flags) {
- return Flags &
- (MachineInstr::MIFlag::FmNoNans | MachineInstr::MIFlag::FmNoInfs |
- MachineInstr::MIFlag::FmNsz | MachineInstr::MIFlag::FmArcp |
- MachineInstr::MIFlag::FmContract | MachineInstr::MIFlag::FmAfn |
- MachineInstr::MIFlag::FmReassoc);
-}
-
void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
unsigned OpIdx, unsigned ExtOpcode) {
MachineOperand &MO = MI.getOperand(OpIdx);
@@ -2180,7 +2171,7 @@ void LegalizerHelper::widenScalarSrcUsingFPExt(MachineInstr &MI, LLT WideTy,
unsigned OpIdx) {
MachineOperand &MO = MI.getOperand(OpIdx);
auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, {WideTy}, {MO},
- extractFMF(MI.getFlags()));
+ MI.getFlags());
MO.setReg(ExtB.getReg(0));
}
@@ -2205,8 +2196,7 @@ void LegalizerHelper::widenScalarDstUsingFPTrunc(MachineInstr &MI, LLT WideTy,
MachineOperand &MO = MI.getOperand(OpIdx);
Register DstExt = MRI.createGenericVirtualRegister(WideTy);
MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
- MIRBuilder.buildInstr(TargetOpcode::G_FPTRUNC, {MO}, {DstExt},
- extractFMF(MI.getFlags()));
+ MIRBuilder.buildInstr(TargetOpcode::G_FPTRUNC, {MO}, {DstExt}, MI.getFlags());
MO.setReg(DstExt);
}
More information about the llvm-commits
mailing list