[llvm] [AArch64][GlobalISel] BF16 sitofp. (PR #210282)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 02:19:42 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/210282
This adds lowering for bf16 s/uitofp. It needs to be careful not to double-round the result and uses lower to perform the correct action based on the size of the input. This does not yet handle i64 which are more involved.
>From 201f5cfbddc25948a8eb8d02f9ec0e38329b4b5b Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Fri, 17 Jul 2026 08:16:55 +0100
Subject: [PATCH 1/2] [AArch64] Attempt to fix vector bf16 itofp converts.
I was looking into supporting BF16 converts in GISel, and in the process ran
into the SDAG code not being correct. The scalar and vector code can produce
different results, as the scalar carefully attempts to avoid double-rounding
issues where as the vector just converts to a f32. A number of other backends
are probably incorrect too.
This corrects that using the existing scalar code for vectors too. A number of
fixups are needs to make sure that the supported types are all handled
correctly through legalization. One change is that because f64->bf16 are not
handled under SVE1 there is no longer a working lowering until that is added.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 307 +++---
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 1 +
.../CodeGen/AArch64/bf16-v4-instructions.ll | 60 +-
.../CodeGen/AArch64/bf16-v8-instructions.ll | 120 ++-
llvm/test/CodeGen/AArch64/itofp-bf16.ll | 986 +++++++++++++-----
.../CodeGen/AArch64/sve-bf16-int-converts.ll | 274 ++++-
6 files changed, 1274 insertions(+), 474 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c2956226e4a92..17307f64fad7c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4921,13 +4921,20 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
if (VT.isScalableVector()) {
// Let common code split the operation.
- if (SrcVT == MVT::nxv8f32)
- return Op;
+ SDLoc DL(Op);
+ if (!isTypeLegal(SrcVT)) {
+ auto [OpLo, OpHi] = DAG.SplitVector(SrcVal, DL);
+ auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
+ OpLo = DAG.getNode(Op.getOpcode(), DL, LoVT, OpLo, Op.getOperand(1),
+ Op->getFlags());
+ OpHi = DAG.getNode(Op.getOpcode(), DL, HiVT, OpHi, Op.getOperand(1),
+ Op->getFlags());
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, OpLo, OpHi);
+ }
if (VT.getScalarType() != MVT::bf16)
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
- SDLoc DL(Op);
constexpr EVT I32 = MVT::nxv4i32;
auto ImmV = [&](int I) -> SDValue { return DAG.getConstant(I, DL, I32); };
@@ -5003,7 +5010,7 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
NaN = DAG.getNode(ISD::OR, DL, I32, Narrow,
DAG.getConstant(0x400000, DL, I32));
}
- } else if (SrcVT.getScalarType() == MVT::f64) {
+ } else if (isTypeLegal(SrcVT) && SrcVT.getScalarType() == MVT::f64) {
Narrow = DAG.getNode(AArch64ISD::FCVTXN, DL, F32, Narrow);
Narrow = DAG.getNode(ISD::BITCAST, DL, I32, Narrow);
} else {
@@ -5025,7 +5032,13 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
SDValue IsNaN = DAG.getSetCC(
DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT),
SrcVal, SrcVal, ISD::SETUO);
- Narrow = DAG.getSelect(DL, I32, IsNaN, NaN, Narrow);
+ Narrow =
+ SrcVT.isFixedLengthVector()
+ ? DAG.getNode(ISD::OR, DL, I32,
+ DAG.getNode(ISD::AND, DL, I32, IsNaN, NaN),
+ DAG.getNode(ISD::AND, DL, I32,
+ DAG.getNOT(DL, IsNaN, I32), Narrow))
+ : DAG.getSelect(DL, I32, IsNaN, NaN, Narrow);
}
// Now that we have rounded, shift the bits into position.
@@ -5416,26 +5429,15 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
return DAG.getNode(ISD::VSELECT, DL, VT, In, TrueVal, FalseVal);
}
- // Promote bf16 conversions to f32.
- if (VT.getVectorElementType() == MVT::bf16) {
- EVT F32 = VT.changeElementType(*DAG.getContext(), MVT::f32);
- if (IsStrict) {
- SDValue Val = DAG.getNode(Op.getOpcode(), DL, {F32, MVT::Other},
- {Op.getOperand(0), In});
- return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
- {Op.getValueType(), MVT::Other},
- {Val.getValue(1), Val.getValue(0),
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
- }
- return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
- DAG.getNode(Op.getOpcode(), DL, F32, In),
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
- }
-
if (VT.isScalableVector()) {
- // Let common code split the operation.
- if (VT == MVT::nxv8f32)
- return Op;
+ // Split illegal operations in half
+ if (!isTypeLegal(VT)) {
+ auto [OpLo, OpHi] = DAG.SplitVector(In, DL);
+ auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
+ OpLo = DAG.getNode(Op.getOpcode(), DL, LoVT, OpLo, Op->getFlags());
+ OpHi = DAG.getNode(Op.getOpcode(), DL, HiVT, OpHi, Op->getFlags());
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, OpLo, OpHi);
+ }
unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
: AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
@@ -5511,138 +5513,157 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
return Op;
}
-SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
- SelectionDAG &DAG) const {
- if (Op.getValueType().isVector())
- return LowerVectorINT_TO_FP(Op, DAG);
-
+static SDValue LowerIntToFpViaPromotion(SDValue Op, EVT PromoteVT,
+ SelectionDAG &DAG) {
+ SDLoc DL(Op);
bool IsStrict = Op->isStrictFPOpcode();
SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
+ PromoteVT = Op.getValueType().changeElementType(*DAG.getContext(), PromoteVT);
+ if (IsStrict) {
+ SDValue Val = DAG.getNode(Op.getOpcode(), DL, {PromoteVT, MVT::Other},
+ {Op.getOperand(0), SrcVal});
+ return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
+ {Op.getValueType(), MVT::Other},
+ {Val.getValue(1), Val.getValue(0),
+ DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
+ }
+ return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
+ DAG.getNode(Op.getOpcode(), DL, PromoteVT, SrcVal),
+ DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
+}
- bool IsSigned = Op->getOpcode() == ISD::STRICT_SINT_TO_FP ||
- Op->getOpcode() == ISD::SINT_TO_FP;
+SDValue AArch64TargetLowering::LowerBF16INT_TO_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ bool IsStrict = Op->isStrictFPOpcode();
+ bool IsSigned = Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
+ Op.getOpcode() == ISD::SINT_TO_FP;
+ SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
+ EVT DstVT = Op.getValueType();
- auto IntToFpViaPromotion = [&](EVT PromoteVT) {
+ unsigned MaxWidth = IsSigned
+ ? DAG.ComputeMaxSignificantBits(SrcVal)
+ : DAG.computeKnownBits(SrcVal).countMaxActiveBits();
+ // bf16 conversions are promoted to f32 when converting from i16.
+ if (MaxWidth <= 24)
+ return LowerIntToFpViaPromotion(Op, MVT::f32, DAG);
+
+ // bf16 conversions are promoted to f64 when converting from i32.
+ if (MaxWidth <= 53)
+ return LowerIntToFpViaPromotion(Op, MVT::f64, DAG);
+
+ // We need to be careful about i64 -> bf16.
+ // Consider an i32 22216703.
+ // This number cannot be represented exactly as an f32 and so a itofp will
+ // turn it into 22216704.0 fptrunc to bf16 will turn this into 22282240.0
+ // However, the correct bf16 was supposed to be 22151168.0
+ // We need to use sticky rounding to get this correct.
+ if (SrcVal.getValueType().getScalarType() == MVT::i64) {
SDLoc DL(Op);
- if (IsStrict) {
- SDValue Val = DAG.getNode(Op.getOpcode(), DL, {PromoteVT, MVT::Other},
- {Op.getOperand(0), SrcVal});
- return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
- {Op.getValueType(), MVT::Other},
- {Val.getValue(1), Val.getValue(0),
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
- }
- return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
- DAG.getNode(Op.getOpcode(), DL, PromoteVT, SrcVal),
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
- };
-
- if (Op.getValueType() == MVT::bf16) {
- unsigned MaxWidth = IsSigned
- ? DAG.ComputeMaxSignificantBits(SrcVal)
- : DAG.computeKnownBits(SrcVal).countMaxActiveBits();
- // bf16 conversions are promoted to f32 when converting from i16.
- if (MaxWidth <= 24) {
- return IntToFpViaPromotion(MVT::f32);
- }
-
- // bf16 conversions are promoted to f64 when converting from i32.
- if (MaxWidth <= 53) {
- return IntToFpViaPromotion(MVT::f64);
- }
+ // This algorithm is equivalent to the following:
+ // uint64_t SrcHi = SrcVal & ~0xfffull;
+ // uint64_t SrcLo = SrcVal & 0xfffull;
+ // uint64_t Highest = SrcVal >> 53;
+ // bool HasHighest = Highest != 0;
+ // uint64_t ToRound = HasHighest ? SrcHi : SrcVal;
+ // double Rounded = static_cast<double>(ToRound);
+ // uint64_t RoundedBits = std::bit_cast<uint64_t>(Rounded);
+ // uint64_t HasLo = SrcLo != 0;
+ // bool NeedsAdjustment = HasHighest & HasLo;
+ // uint64_t AdjustedBits = RoundedBits | uint64_t{NeedsAdjustment};
+ // double Adjusted = std::bit_cast<double>(AdjustedBits);
+ // return static_cast<__bf16>(Adjusted);
+ //
+ // Essentially, what happens is that SrcVal either fits perfectly in a
+ // double-precision value or it is too big. If it is sufficiently small,
+ // we should just go u64 -> double -> bf16 in a naive way. Otherwise, we
+ // ensure that u64 -> double has no rounding error by only using the 52
+ // MSB of the input. The low order bits will get merged into a sticky bit
+ // which will avoid issues incurred by double rounding.
+
+ // Signed conversion is more or less like so:
+ // copysign((__bf16)abs(SrcVal), SrcVal)
+ EVT i64Ty = DstVT.changeElementType(*DAG.getContext(), MVT::i64);
+ EVT f64Ty = DstVT.changeElementType(*DAG.getContext(), MVT::f64);
+ SDValue SignBit;
+ if (IsSigned) {
+ SignBit = DAG.getNode(ISD::AND, DL, i64Ty, SrcVal,
+ DAG.getConstant(1ull << 63, DL, i64Ty));
+ SrcVal = DAG.getNode(ISD::ABS, DL, i64Ty, SrcVal);
+ }
+ SDValue SrcHi = DAG.getNode(ISD::AND, DL, i64Ty, SrcVal,
+ DAG.getConstant(~0xfffull, DL, i64Ty));
+ SDValue SrcLo = DAG.getNode(ISD::AND, DL, i64Ty, SrcVal,
+ DAG.getConstant(0xfffull, DL, i64Ty));
+ SDValue Highest = DAG.getNode(ISD::SRL, DL, i64Ty, SrcVal,
+ DAG.getShiftAmountConstant(53, i64Ty, DL));
+ SDValue Zero64 = DAG.getConstant(0, DL, i64Ty);
+ SDValue ToRound =
+ DAG.getSelectCC(DL, Highest, Zero64, SrcHi, SrcVal, ISD::SETNE);
+ SDValue Rounded = IsStrict
+ ? DAG.getNode(Op.getOpcode(), DL, {f64Ty, MVT::Other},
+ {Op.getOperand(0), ToRound})
+ : DAG.getNode(Op.getOpcode(), DL, f64Ty, ToRound);
+
+ SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, i64Ty, Rounded);
+ if (SignBit) {
+ RoundedBits = DAG.getNode(ISD::OR, DL, i64Ty, RoundedBits, SignBit);
+ }
+
+ SDValue HasHighest = DAG.getSetCC(
+ DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), i64Ty),
+ Highest, Zero64, ISD::SETNE);
+
+ SDValue HasLo = DAG.getSetCC(
+ DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), i64Ty),
+ SrcLo, Zero64, ISD::SETNE);
+
+ SDValue NeedsAdjustment =
+ DAG.getNode(ISD::AND, DL, HasLo.getValueType(), HasHighest, HasLo);
+ NeedsAdjustment = DAG.getZExtOrTrunc(NeedsAdjustment, DL, i64Ty);
+
+ SDValue AdjustedBits =
+ DAG.getNode(ISD::OR, DL, i64Ty, RoundedBits, NeedsAdjustment);
+ SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, f64Ty, AdjustedBits);
+ return IsStrict
+ ? DAG.getNode(ISD::STRICT_FP_ROUND, DL,
+ {Op.getValueType(), MVT::Other},
+ {Rounded.getValue(1), Adjusted,
+ DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)})
+ : DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), Adjusted,
+ DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
+ }
- // We need to be careful about i64 -> bf16.
- // Consider an i32 22216703.
- // This number cannot be represented exactly as an f32 and so a itofp will
- // turn it into 22216704.0 fptrunc to bf16 will turn this into 22282240.0
- // However, the correct bf16 was supposed to be 22151168.0
- // We need to use sticky rounding to get this correct.
- if (SrcVal.getValueType() == MVT::i64) {
- SDLoc DL(Op);
- // This algorithm is equivalent to the following:
- // uint64_t SrcHi = SrcVal & ~0xfffull;
- // uint64_t SrcLo = SrcVal & 0xfffull;
- // uint64_t Highest = SrcVal >> 53;
- // bool HasHighest = Highest != 0;
- // uint64_t ToRound = HasHighest ? SrcHi : SrcVal;
- // double Rounded = static_cast<double>(ToRound);
- // uint64_t RoundedBits = std::bit_cast<uint64_t>(Rounded);
- // uint64_t HasLo = SrcLo != 0;
- // bool NeedsAdjustment = HasHighest & HasLo;
- // uint64_t AdjustedBits = RoundedBits | uint64_t{NeedsAdjustment};
- // double Adjusted = std::bit_cast<double>(AdjustedBits);
- // return static_cast<__bf16>(Adjusted);
- //
- // Essentially, what happens is that SrcVal either fits perfectly in a
- // double-precision value or it is too big. If it is sufficiently small,
- // we should just go u64 -> double -> bf16 in a naive way. Otherwise, we
- // ensure that u64 -> double has no rounding error by only using the 52
- // MSB of the input. The low order bits will get merged into a sticky bit
- // which will avoid issues incurred by double rounding.
-
- // Signed conversion is more or less like so:
- // copysign((__bf16)abs(SrcVal), SrcVal)
- SDValue SignBit;
- if (IsSigned) {
- SignBit = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
- DAG.getConstant(1ull << 63, DL, MVT::i64));
- SrcVal = DAG.getNode(ISD::ABS, DL, MVT::i64, SrcVal);
- }
- SDValue SrcHi = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
- DAG.getConstant(~0xfffull, DL, MVT::i64));
- SDValue SrcLo = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
- DAG.getConstant(0xfffull, DL, MVT::i64));
- SDValue Highest =
- DAG.getNode(ISD::SRL, DL, MVT::i64, SrcVal,
- DAG.getShiftAmountConstant(53, MVT::i64, DL));
- SDValue Zero64 = DAG.getConstant(0, DL, MVT::i64);
- SDValue ToRound =
- DAG.getSelectCC(DL, Highest, Zero64, SrcHi, SrcVal, ISD::SETNE);
- SDValue Rounded =
- IsStrict ? DAG.getNode(Op.getOpcode(), DL, {MVT::f64, MVT::Other},
- {Op.getOperand(0), ToRound})
- : DAG.getNode(Op.getOpcode(), DL, MVT::f64, ToRound);
-
- SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Rounded);
- if (SignBit) {
- RoundedBits = DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, SignBit);
- }
+ return SDValue();
+}
- SDValue HasHighest = DAG.getSetCC(
- DL,
- getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
- Highest, Zero64, ISD::SETNE);
+SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ // NOTE: i1->bf16 does not require promotion to f32.
+ EVT VT = Op.getValueType();
+ SDValue Src = Op.getOperand(Op->isStrictFPOpcode() ? 1 : 0);
+ EVT SrcVT = Src.getValueType();
+ bool IsSigned = Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
+ Op.getOpcode() == ISD::SINT_TO_FP;
- SDValue HasLo = DAG.getSetCC(
- DL,
- getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
- SrcLo, Zero64, ISD::SETNE);
-
- SDValue NeedsAdjustment =
- DAG.getNode(ISD::AND, DL, HasLo.getValueType(), HasHighest, HasLo);
- NeedsAdjustment = DAG.getZExtOrTrunc(NeedsAdjustment, DL, MVT::i64);
-
- SDValue AdjustedBits =
- DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, NeedsAdjustment);
- SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, MVT::f64, AdjustedBits);
- return IsStrict
- ? DAG.getNode(
- ISD::STRICT_FP_ROUND, DL,
- {Op.getValueType(), MVT::Other},
- {Rounded.getValue(1), Adjusted,
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)})
- : DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), Adjusted,
- DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
- }
+ if (VT.isScalableVector() && SrcVT.getVectorElementType() == MVT::i1) {
+ SDLoc DL(Op);
+ SDValue FalseVal = DAG.getConstantFP(0.0, DL, VT);
+ SDValue TrueVal = IsSigned ? DAG.getConstantFP(-1.0, DL, VT)
+ : DAG.getConstantFP(1.0, DL, VT);
+ return DAG.getNode(ISD::VSELECT, DL, VT, Src, TrueVal, FalseVal);
}
+ if (Op.getValueType().getScalarType() == MVT::bf16)
+ return LowerBF16INT_TO_FP(Op, DAG);
+ if (Op.getValueType().isVector())
+ return LowerVectorINT_TO_FP(Op, DAG);
+
// f16 conversions are promoted to f32 when full fp16 is not supported.
- if (Op.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
- return IntToFpViaPromotion(MVT::f32);
- }
+ if (Op.getValueType() == MVT::f16 && !Subtarget->hasFullFP16())
+ return LowerIntToFpViaPromotion(Op, MVT::f32, DAG);
// i128 conversions are libcalls.
- if (SrcVal.getValueType() == MVT::i128)
+ if (SrcVT == MVT::i128)
return SDValue();
// Other conversions are legal, unless it's to the completely software-based
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 5cc4e9fa1b063..e07f7d4640a5c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -793,6 +793,7 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVectorXRINT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBF16INT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index ea262467e66fe..211353482a8bb 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -1624,19 +1624,33 @@ define <4 x bfloat> @test_sitofp_i16(<4 x i16> %a) #0 {
define <4 x bfloat> @test_sitofp_i32(<4 x i32> %a) #0 {
; CHECK-CVT-LABEL: test_sitofp_i32:
; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-CVT-NEXT: sshll2 v0.2d, v0.4s, #0
; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-NEXT: movi v0.4s, #1
+; CHECK-CVT-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-CVT-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i32:
; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-BF16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-BF16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-BF16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-BF16-NEXT: ret
%1 = sitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %1
@@ -1738,19 +1752,33 @@ define <4 x bfloat> @test_uitofp_i16(<4 x i16> %a) #0 {
define <4 x bfloat> @test_uitofp_i32(<4 x i32> %a) #0 {
; CHECK-CVT-LABEL: test_uitofp_i32:
; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-CVT-NEXT: ushll2 v0.2d, v0.4s, #0
; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-NEXT: movi v0.4s, #1
+; CHECK-CVT-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-CVT-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i32:
; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-BF16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-BF16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-BF16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-BF16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-BF16-NEXT: ret
%1 = uitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %1
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 85faf1facc264..6ec3c71c4d6dd 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -3785,26 +3785,54 @@ define <8 x bfloat> @test_sitofp_i16(<8 x i16> %a) #0 {
define <8 x bfloat> @test_sitofp_i32(<8 x i32> %a) #0 {
; CHECK-CVT-LABEL: test_sitofp_i32:
; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: scvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-NEXT: sshll v2.2d, v1.2s, #0
+; CHECK-CVT-NEXT: sshll v3.2d, v0.2s, #0
+; CHECK-CVT-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-CVT-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-CVT-NEXT: scvtf v2.2d, v2.2d
+; CHECK-CVT-NEXT: scvtf v3.2d, v3.2d
+; CHECK-CVT-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-CVT-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-CVT-NEXT: movi v0.4s, #1
+; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-CVT-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-CVT-NEXT: mov v1.16b, v5.16b
+; CHECK-CVT-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-CVT-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-CVT-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i32:
; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-BF16-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-BF16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-BF16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-BF16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-BF16-NEXT: scvtf v3.2d, v3.2d
+; CHECK-BF16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-NEXT: scvtf v0.2d, v1.2d
+; CHECK-BF16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
; CHECK-BF16-NEXT: ret
%1 = sitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %1
@@ -4009,26 +4037,54 @@ define <8 x bfloat> @test_uitofp_i16(<8 x i16> %a) #0 {
define <8 x bfloat> @test_uitofp_i32(<8 x i32> %a) #0 {
; CHECK-CVT-LABEL: test_uitofp_i32:
; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-NEXT: ushll v2.2d, v1.2s, #0
+; CHECK-CVT-NEXT: ushll v3.2d, v0.2s, #0
+; CHECK-CVT-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-CVT-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-CVT-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-CVT-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-CVT-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-CVT-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-CVT-NEXT: movi v0.4s, #1
+; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-CVT-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-CVT-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-CVT-NEXT: mov v1.16b, v5.16b
+; CHECK-CVT-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-CVT-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-CVT-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i32:
; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-BF16-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-BF16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-BF16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-BF16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-BF16-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-BF16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-BF16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-BF16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
; CHECK-BF16-NEXT: ret
%1 = uitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %1
diff --git a/llvm/test/CodeGen/AArch64/itofp-bf16.ll b/llvm/test/CodeGen/AArch64/itofp-bf16.ll
index 77d0c318f0fc4..90904cec47722 100644
--- a/llvm/test/CodeGen/AArch64/itofp-bf16.ll
+++ b/llvm/test/CodeGen/AArch64/itofp-bf16.ll
@@ -1675,21 +1675,36 @@ entry:
define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v2i32_v2bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: movi v1.2d, #0000000000000000
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NOFP16-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-NEXT: bit v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v2i32_v2bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-NEXT: movi d1, #0000000000000000
+; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn s2, d0
+; CHECK-FP16-NEXT: mov d0, v0.d[1]
+; CHECK-FP16-NEXT: bfcvt h2, s2
+; CHECK-FP16-NEXT: fcvtxn s0, d0
+; CHECK-FP16-NEXT: mov v1.h[0], v2.h[0]
+; CHECK-FP16-NEXT: bfcvt h0, s0
+; CHECK-FP16-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-FP16-NEXT: fmov d0, d1
; CHECK-FP16-NEXT: ret
entry:
%c = sitofp <2 x i32> %a to <2 x bfloat>
@@ -1699,21 +1714,36 @@ entry:
define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v2i32_v2bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: movi v1.2d, #0000000000000000
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NOFP16-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-NEXT: bit v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v2i32_v2bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-NEXT: movi d1, #0000000000000000
+; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn s2, d0
+; CHECK-FP16-NEXT: mov d0, v0.d[1]
+; CHECK-FP16-NEXT: bfcvt h2, s2
+; CHECK-FP16-NEXT: fcvtxn s0, d0
+; CHECK-FP16-NEXT: mov v1.h[0], v2.h[0]
+; CHECK-FP16-NEXT: bfcvt h0, s0
+; CHECK-FP16-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-FP16-NEXT: fmov d0, d1
; CHECK-FP16-NEXT: ret
entry:
%c = uitofp <2 x i32> %a to <2 x bfloat>
@@ -1723,19 +1753,33 @@ entry:
define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v3i32_v3bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v3i32_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
entry:
%c = sitofp <3 x i32> %a to <3 x bfloat>
@@ -1745,19 +1789,33 @@ entry:
define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v3i32_v3bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v3i32_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
entry:
%c = uitofp <3 x i32> %a to <3 x bfloat>
@@ -1767,19 +1825,33 @@ entry:
define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v4i32_v4bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v4i32_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
entry:
%c = sitofp <4 x i32> %a to <4 x bfloat>
@@ -1789,19 +1861,33 @@ entry:
define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i32_v4bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v4i32_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
entry:
%c = uitofp <4 x i32> %a to <4 x bfloat>
@@ -1811,26 +1897,54 @@ entry:
define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v8i32_v8bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: movi v5.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-NOFP16-NEXT: sshll v2.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: sshll v3.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NOFP16-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: mov v1.16b, v5.16b
+; CHECK-NOFP16-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-NOFP16-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v8i32_v8bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-FP16-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
; CHECK-FP16-NEXT: ret
entry:
%c = sitofp <8 x i32> %a to <8 x bfloat>
@@ -1840,26 +1954,54 @@ entry:
define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v8i32_v8bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: movi v5.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-NOFP16-NEXT: ushll v2.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: ushll v3.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NOFP16-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: mov v1.16b, v5.16b
+; CHECK-NOFP16-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-NOFP16-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v8i32_v8bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-FP16-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
; CHECK-FP16-NEXT: ret
entry:
%c = uitofp <8 x i32> %a to <8 x bfloat>
@@ -1869,39 +2011,94 @@ entry:
define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v16i32_v16bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: movi v17.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v6.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v7.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v6.16b, v6.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v7.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v6.16b, v16.16b, v1.16b
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v17.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v2.4s, v17.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v5.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v6.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v2.4s, v17.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v3.4s, v17.4s
+; CHECK-NOFP16-NEXT: sshll v5.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: sshll v4.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: sshll v6.2d, v3.2s, #0
+; CHECK-NOFP16-NEXT: sshll v7.2d, v2.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-NEXT: scvtf v5.2d, v5.2d
+; CHECK-NOFP16-NEXT: scvtf v4.2d, v4.2d
+; CHECK-NOFP16-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-NOFP16-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-NOFP16-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-NOFP16-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v5.4s, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v6.4s, v3.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v7.4s, v2.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-NOFP16-NEXT: ushr v2.4s, v4.4s, #16
+; CHECK-NOFP16-NEXT: ushr v16.4s, v6.4s, #16
+; CHECK-NOFP16-NEXT: ushr v17.4s, v7.4s, #16
+; CHECK-NOFP16-NEXT: add v19.4s, v5.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v18.4s, v4.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v20.4s, v6.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v17.4s, v4.4s, v4.4s
+; CHECK-NOFP16-NEXT: orr v4.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v3.4s, v19.4s, v3.4s
+; CHECK-NOFP16-NEXT: fcmeq v19.4s, v6.4s, v6.4s
+; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
+; CHECK-NOFP16-NEXT: fcmeq v18.4s, v5.4s, v5.4s
+; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-NEXT: fcmeq v1.4s, v7.4s, v7.4s
+; CHECK-NOFP16-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-NEXT: orr v5.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v6.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v7.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: bif v2.16b, v4.16b, v17.16b
+; CHECK-NOFP16-NEXT: mov v4.16b, v19.16b
+; CHECK-NOFP16-NEXT: bif v3.16b, v5.16b, v18.16b
+; CHECK-NOFP16-NEXT: bsl v1.16b, v0.16b, v7.16b
+; CHECK-NOFP16-NEXT: bsl v4.16b, v16.16b, v6.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v3.8h, v2.8h
+; CHECK-NOFP16-NEXT: uzp2 v1.8h, v1.8h, v4.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v16i32_v16bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: scvtf v4.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: scvtf v2.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-NEXT: sshll v4.2d, v0.2s, #0
+; CHECK-FP16-NEXT: sshll v5.2d, v2.2s, #0
+; CHECK-FP16-NEXT: sshll v6.2d, v1.2s, #0
+; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: sshll v7.2d, v3.2s, #0
+; CHECK-FP16-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v2.2s, v7.2d
+; CHECK-FP16-NEXT: scvtf v1.2d, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v6.4s
; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT: ret
entry:
@@ -1912,39 +2109,94 @@ entry:
define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v16i32_v16bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: movi v17.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v6.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v7.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v6.16b, v6.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v7.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v6.16b, v16.16b, v1.16b
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v17.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v2.4s, v17.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v5.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v6.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v2.4s, v17.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v3.4s, v17.4s
+; CHECK-NOFP16-NEXT: ushll v5.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: ushll v4.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: ushll v6.2d, v3.2s, #0
+; CHECK-NOFP16-NEXT: ushll v7.2d, v2.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-NOFP16-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-NOFP16-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-NOFP16-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v5.4s, v0.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v6.4s, v3.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v7.4s, v2.2d
+; CHECK-NOFP16-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-NOFP16-NEXT: ushr v2.4s, v4.4s, #16
+; CHECK-NOFP16-NEXT: ushr v16.4s, v6.4s, #16
+; CHECK-NOFP16-NEXT: ushr v17.4s, v7.4s, #16
+; CHECK-NOFP16-NEXT: add v19.4s, v5.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v18.4s, v4.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v20.4s, v6.4s, v1.4s
+; CHECK-NOFP16-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-NEXT: and v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-NEXT: fcmeq v17.4s, v4.4s, v4.4s
+; CHECK-NOFP16-NEXT: orr v4.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v3.4s, v19.4s, v3.4s
+; CHECK-NOFP16-NEXT: fcmeq v19.4s, v6.4s, v6.4s
+; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
+; CHECK-NOFP16-NEXT: fcmeq v18.4s, v5.4s, v5.4s
+; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-NEXT: fcmeq v1.4s, v7.4s, v7.4s
+; CHECK-NOFP16-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-NEXT: orr v5.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v6.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v7.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: bif v2.16b, v4.16b, v17.16b
+; CHECK-NOFP16-NEXT: mov v4.16b, v19.16b
+; CHECK-NOFP16-NEXT: bif v3.16b, v5.16b, v18.16b
+; CHECK-NOFP16-NEXT: bsl v1.16b, v0.16b, v7.16b
+; CHECK-NOFP16-NEXT: bsl v4.16b, v16.16b, v6.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v3.8h, v2.8h
+; CHECK-NOFP16-NEXT: uzp2 v1.8h, v1.8h, v4.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v16i32_v16bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: ucvtf v4.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-NEXT: ushll v4.2d, v0.2s, #0
+; CHECK-FP16-NEXT: ushll v5.2d, v2.2s, #0
+; CHECK-FP16-NEXT: ushll v6.2d, v1.2s, #0
+; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: ushll v7.2d, v3.2s, #0
+; CHECK-FP16-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-NEXT: fcvtxn v2.2s, v7.2d
+; CHECK-FP16-NEXT: ucvtf v1.2d, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v0.2d
+; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v6.4s
; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT: ret
entry:
@@ -1955,68 +2207,179 @@ entry:
define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v32i32_v32bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: scvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: movi v16.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: scvtf v17.4s, v3.4s
-; CHECK-NOFP16-NEXT: scvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: scvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v18.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v19.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v20.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v18.16b, v18.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v19.16b, v19.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v20.16b, v20.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: and v3.16b, v22.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v19.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v20.4s, v6.4s
-; CHECK-NOFP16-NEXT: and v18.16b, v23.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v19.16b, v24.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v25.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v20.4s, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v2.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v4.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v6.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v18.4s, v17.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v19.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v20.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v4.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v5.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v6.4s, v21.4s
+; CHECK-NOFP16-NEXT: sshll v17.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: sshll v18.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v19.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: sshll v16.2d, v3.2s, #0
+; CHECK-NOFP16-NEXT: sshll v21.2d, v2.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-NEXT: sshll v23.2d, v7.2s, #0
+; CHECK-NOFP16-NEXT: sshll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-NEXT: scvtf v17.2d, v17.2d
+; CHECK-NOFP16-NEXT: scvtf v18.2d, v18.2d
+; CHECK-NOFP16-NEXT: scvtf v20.2d, v1.2d
+; CHECK-NOFP16-NEXT: scvtf v19.2d, v19.2d
+; CHECK-NOFP16-NEXT: scvtf v16.2d, v16.2d
+; CHECK-NOFP16-NEXT: scvtf v22.2d, v3.2d
+; CHECK-NOFP16-NEXT: scvtf v23.2d, v23.2d
+; CHECK-NOFP16-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v17.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v18.2d
+; CHECK-NOFP16-NEXT: sshll2 v18.2d, v2.4s, #0
+; CHECK-NOFP16-NEXT: scvtf v2.2d, v21.2d
+; CHECK-NOFP16-NEXT: sshll v21.2d, v4.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v16.2d
+; CHECK-NOFP16-NEXT: sshll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-NEXT: movi v17.4s, #1
+; CHECK-NOFP16-NEXT: movi v16.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: scvtf v18.2d, v18.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v20.2d
+; CHECK-NOFP16-NEXT: sshll v20.2d, v5.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v19.2d
+; CHECK-NOFP16-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-NEXT: scvtf v21.2d, v21.2d
+; CHECK-NOFP16-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-NEXT: sshll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v22.2d
+; CHECK-NOFP16-NEXT: scvtf v20.2d, v20.2d
+; CHECK-NOFP16-NEXT: scvtf v4.2d, v4.2d
+; CHECK-NOFP16-NEXT: scvtf v19.2d, v19.2d
+; CHECK-NOFP16-NEXT: ushr v22.4s, v0.4s, #16
+; CHECK-NOFP16-NEXT: add v24.4s, v0.4s, v16.4s
+; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v18.2d
+; CHECK-NOFP16-NEXT: scvtf v5.2d, v5.2d
+; CHECK-NOFP16-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-NEXT: fcmeq v27.4s, v3.4s, v3.4s
+; CHECK-NOFP16-NEXT: fcvtxn v18.2s, v20.2d
+; CHECK-NOFP16-NEXT: fcvtxn v20.2s, v21.2d
+; CHECK-NOFP16-NEXT: fcvtxn v21.2s, v23.2d
+; CHECK-NOFP16-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v17.16b
+; CHECK-NOFP16-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: fcvtxn2 v18.4s, v5.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v21.4s, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v19.4s, v6.2d
+; CHECK-NOFP16-NEXT: add v22.4s, v24.4s, v22.4s
+; CHECK-NOFP16-NEXT: and v24.16b, v25.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v25.4s, v1.4s, v16.4s
+; CHECK-NOFP16-NEXT: and v5.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: ushr v26.4s, v2.4s, #16
+; CHECK-NOFP16-NEXT: add v4.4s, v3.4s, v16.4s
+; CHECK-NOFP16-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: ushr v28.4s, v21.4s, #16
+; CHECK-NOFP16-NEXT: add v30.4s, v20.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v31.4s, v21.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v6.4s, v25.4s, v24.4s
+; CHECK-NOFP16-NEXT: and v24.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v25.4s, v2.4s, v16.4s
+; CHECK-NOFP16-NEXT: ushr v26.4s, v18.4s, #16
+; CHECK-NOFP16-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-NEXT: ushr v5.4s, v20.4s, #16
+; CHECK-NOFP16-NEXT: ushr v29.4s, v19.4s, #16
+; CHECK-NOFP16-NEXT: and v28.16b, v28.16b, v17.16b
+; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v24.4s, v25.4s, v24.4s
+; CHECK-NOFP16-NEXT: bit v1.16b, v6.16b, v7.16b
+; CHECK-NOFP16-NEXT: bit v0.16b, v22.16b, v23.16b
+; CHECK-NOFP16-NEXT: and v25.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v26.4s, v18.4s, v16.4s
+; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v17.16b
+; CHECK-NOFP16-NEXT: and v17.16b, v29.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v16.4s, v19.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v28.4s, v31.4s, v28.4s
+; CHECK-NOFP16-NEXT: fcmeq v31.4s, v21.4s, v21.4s
+; CHECK-NOFP16-NEXT: fcmeq v29.4s, v2.4s, v2.4s
+; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v25.4s, v26.4s, v25.4s
+; CHECK-NOFP16-NEXT: fcmeq v26.4s, v18.4s, v18.4s
+; CHECK-NOFP16-NEXT: add v5.4s, v30.4s, v5.4s
+; CHECK-NOFP16-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-NOFP16-NEXT: fcmeq v17.4s, v19.4s, v19.4s
+; CHECK-NOFP16-NEXT: fcmeq v30.4s, v20.4s, v20.4s
+; CHECK-NOFP16-NEXT: orr v18.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v20.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v21.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v19.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: bit v3.16b, v4.16b, v27.16b
+; CHECK-NOFP16-NEXT: mov v6.16b, v31.16b
+; CHECK-NOFP16-NEXT: mov v4.16b, v26.16b
+; CHECK-NOFP16-NEXT: bit v2.16b, v24.16b, v29.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-NEXT: mov v7.16b, v17.16b
+; CHECK-NOFP16-NEXT: bif v5.16b, v20.16b, v30.16b
+; CHECK-NOFP16-NEXT: bsl v6.16b, v28.16b, v21.16b
+; CHECK-NOFP16-NEXT: bsl v4.16b, v25.16b, v18.16b
+; CHECK-NOFP16-NEXT: bsl v7.16b, v16.16b, v19.16b
+; CHECK-NOFP16-NEXT: uzp2 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-NEXT: uzp2 v2.8h, v5.8h, v4.8h
+; CHECK-NOFP16-NEXT: uzp2 v3.8h, v7.8h, v6.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v32i32_v32bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-FP16-NEXT: scvtf v6.4s, v6.4s
-; CHECK-FP16-NEXT: scvtf v16.4s, v1.4s
-; CHECK-FP16-NEXT: scvtf v17.4s, v3.4s
-; CHECK-FP16-NEXT: scvtf v5.4s, v5.4s
-; CHECK-FP16-NEXT: scvtf v7.4s, v7.4s
+; CHECK-FP16-NEXT: sshll v17.2d, v0.2s, #0
+; CHECK-FP16-NEXT: sshll v16.2d, v2.2s, #0
+; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: sshll v19.2d, v4.2s, #0
+; CHECK-FP16-NEXT: sshll v20.2d, v6.2s, #0
+; CHECK-FP16-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-NEXT: sshll v21.2d, v3.2s, #0
+; CHECK-FP16-NEXT: sshll v22.2d, v5.2s, #0
+; CHECK-FP16-NEXT: sshll2 v4.2d, v4.4s, #0
+; CHECK-FP16-NEXT: scvtf v18.2d, v17.2d
+; CHECK-FP16-NEXT: scvtf v17.2d, v0.2d
+; CHECK-FP16-NEXT: scvtf v16.2d, v16.2d
+; CHECK-FP16-NEXT: scvtf v19.2d, v19.2d
+; CHECK-FP16-NEXT: scvtf v20.2d, v20.2d
+; CHECK-FP16-NEXT: sshll2 v6.2d, v6.4s, #0
+; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: scvtf v21.2d, v21.2d
+; CHECK-FP16-NEXT: scvtf v22.2d, v22.2d
+; CHECK-FP16-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-FP16-NEXT: fcvtxn v0.2s, v18.2d
+; CHECK-FP16-NEXT: sshll v18.2d, v1.2s, #0
+; CHECK-FP16-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-NEXT: scvtf v18.2d, v18.2d
+; CHECK-FP16-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-FP16-NEXT: sshll v17.2d, v7.2s, #0
+; CHECK-FP16-NEXT: fcvtxn2 v16.4s, v2.2d
+; CHECK-FP16-NEXT: sshll2 v2.2d, v7.4s, #0
+; CHECK-FP16-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn v7.2s, v21.2d
+; CHECK-FP16-NEXT: fcvtxn2 v19.4s, v4.2d
+; CHECK-FP16-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-NEXT: fcvtxn2 v20.4s, v6.2d
+; CHECK-FP16-NEXT: scvtf v17.2d, v17.2d
+; CHECK-FP16-NEXT: fcvtxn v18.2s, v18.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v17.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v7.4s
+; CHECK-FP16-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-NEXT: bfcvtn v3.4h, v20.4s
+; CHECK-FP16-NEXT: fcvtxn v6.2s, v17.2d
+; CHECK-FP16-NEXT: scvtf v17.2d, v2.2d
+; CHECK-FP16-NEXT: fcvtxn2 v18.4s, v1.2d
+; CHECK-FP16-NEXT: bfcvtn v1.4h, v16.4s
+; CHECK-FP16-NEXT: bfcvtn v2.4h, v19.4s
+; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v17.2d
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v6.4s
; CHECK-FP16-NEXT: ret
entry:
%c = sitofp <32 x i32> %a to <32 x bfloat>
@@ -2026,68 +2389,179 @@ entry:
define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v32i32_v32bf16:
; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: ucvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: movi v16.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: ucvtf v17.4s, v3.4s
-; CHECK-NOFP16-NEXT: ucvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: ucvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: ushr v18.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v19.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v20.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v18.16b, v18.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v19.16b, v19.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v20.16b, v20.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: and v3.16b, v22.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v19.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v20.4s, v6.4s
-; CHECK-NOFP16-NEXT: and v18.16b, v23.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v19.16b, v24.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v25.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v20.4s, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v2.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v4.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v6.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v18.4s, v17.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v19.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v20.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v4.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v5.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v6.4s, v21.4s
+; CHECK-NOFP16-NEXT: ushll v17.2d, v1.2s, #0
+; CHECK-NOFP16-NEXT: ushll v18.2d, v0.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v19.2d, v0.4s, #0
+; CHECK-NOFP16-NEXT: ushll v16.2d, v3.2s, #0
+; CHECK-NOFP16-NEXT: ushll v21.2d, v2.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-NEXT: ushll v23.2d, v7.2s, #0
+; CHECK-NOFP16-NEXT: ushll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-NOFP16-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-NOFP16-NEXT: ucvtf v20.2d, v1.2d
+; CHECK-NOFP16-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-NOFP16-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-NOFP16-NEXT: ucvtf v22.2d, v3.2d
+; CHECK-NOFP16-NEXT: ucvtf v23.2d, v23.2d
+; CHECK-NOFP16-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v17.2d
+; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v18.2d
+; CHECK-NOFP16-NEXT: ushll2 v18.2d, v2.4s, #0
+; CHECK-NOFP16-NEXT: ucvtf v2.2d, v21.2d
+; CHECK-NOFP16-NEXT: ushll v21.2d, v4.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v16.2d
+; CHECK-NOFP16-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-NEXT: movi v17.4s, #1
+; CHECK-NOFP16-NEXT: movi v16.4s, #127, msl #8
+; CHECK-NOFP16-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v20.2d
+; CHECK-NOFP16-NEXT: ushll v20.2d, v5.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v19.2d
+; CHECK-NOFP16-NEXT: ushll v19.2d, v6.2s, #0
+; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-NOFP16-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v22.2d
+; CHECK-NOFP16-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-NOFP16-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-NOFP16-NEXT: ushr v22.4s, v0.4s, #16
+; CHECK-NOFP16-NEXT: add v24.4s, v0.4s, v16.4s
+; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v18.2d
+; CHECK-NOFP16-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-NEXT: fcmeq v27.4s, v3.4s, v3.4s
+; CHECK-NOFP16-NEXT: fcvtxn v18.2s, v20.2d
+; CHECK-NOFP16-NEXT: fcvtxn v20.2s, v21.2d
+; CHECK-NOFP16-NEXT: fcvtxn v21.2s, v23.2d
+; CHECK-NOFP16-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v17.16b
+; CHECK-NOFP16-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: fcvtxn2 v18.4s, v5.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v21.4s, v7.2d
+; CHECK-NOFP16-NEXT: fcvtxn2 v19.4s, v6.2d
+; CHECK-NOFP16-NEXT: add v22.4s, v24.4s, v22.4s
+; CHECK-NOFP16-NEXT: and v24.16b, v25.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v25.4s, v1.4s, v16.4s
+; CHECK-NOFP16-NEXT: and v5.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: ushr v26.4s, v2.4s, #16
+; CHECK-NOFP16-NEXT: add v4.4s, v3.4s, v16.4s
+; CHECK-NOFP16-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: ushr v28.4s, v21.4s, #16
+; CHECK-NOFP16-NEXT: add v30.4s, v20.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v31.4s, v21.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v6.4s, v25.4s, v24.4s
+; CHECK-NOFP16-NEXT: and v24.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v25.4s, v2.4s, v16.4s
+; CHECK-NOFP16-NEXT: ushr v26.4s, v18.4s, #16
+; CHECK-NOFP16-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-NEXT: ushr v5.4s, v20.4s, #16
+; CHECK-NOFP16-NEXT: ushr v29.4s, v19.4s, #16
+; CHECK-NOFP16-NEXT: and v28.16b, v28.16b, v17.16b
+; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v24.4s, v25.4s, v24.4s
+; CHECK-NOFP16-NEXT: bit v1.16b, v6.16b, v7.16b
+; CHECK-NOFP16-NEXT: bit v0.16b, v22.16b, v23.16b
+; CHECK-NOFP16-NEXT: and v25.16b, v26.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v26.4s, v18.4s, v16.4s
+; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v17.16b
+; CHECK-NOFP16-NEXT: and v17.16b, v29.16b, v17.16b
+; CHECK-NOFP16-NEXT: add v16.4s, v19.4s, v16.4s
+; CHECK-NOFP16-NEXT: add v28.4s, v31.4s, v28.4s
+; CHECK-NOFP16-NEXT: fcmeq v31.4s, v21.4s, v21.4s
+; CHECK-NOFP16-NEXT: fcmeq v29.4s, v2.4s, v2.4s
+; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: add v25.4s, v26.4s, v25.4s
+; CHECK-NOFP16-NEXT: fcmeq v26.4s, v18.4s, v18.4s
+; CHECK-NOFP16-NEXT: add v5.4s, v30.4s, v5.4s
+; CHECK-NOFP16-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-NOFP16-NEXT: fcmeq v17.4s, v19.4s, v19.4s
+; CHECK-NOFP16-NEXT: fcmeq v30.4s, v20.4s, v20.4s
+; CHECK-NOFP16-NEXT: orr v18.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v20.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v21.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: orr v19.4s, #64, lsl #16
+; CHECK-NOFP16-NEXT: bit v3.16b, v4.16b, v27.16b
+; CHECK-NOFP16-NEXT: mov v6.16b, v31.16b
+; CHECK-NOFP16-NEXT: mov v4.16b, v26.16b
+; CHECK-NOFP16-NEXT: bit v2.16b, v24.16b, v29.16b
+; CHECK-NOFP16-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-NEXT: mov v7.16b, v17.16b
+; CHECK-NOFP16-NEXT: bif v5.16b, v20.16b, v30.16b
+; CHECK-NOFP16-NEXT: bsl v6.16b, v28.16b, v21.16b
+; CHECK-NOFP16-NEXT: bsl v4.16b, v25.16b, v18.16b
+; CHECK-NOFP16-NEXT: bsl v7.16b, v16.16b, v19.16b
+; CHECK-NOFP16-NEXT: uzp2 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-NEXT: uzp2 v2.8h, v5.8h, v4.8h
+; CHECK-NOFP16-NEXT: uzp2 v3.8h, v7.8h, v6.8h
; CHECK-NOFP16-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v32i32_v32bf16:
; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-FP16-NEXT: ucvtf v6.4s, v6.4s
-; CHECK-FP16-NEXT: ucvtf v16.4s, v1.4s
-; CHECK-FP16-NEXT: ucvtf v17.4s, v3.4s
-; CHECK-FP16-NEXT: ucvtf v5.4s, v5.4s
-; CHECK-FP16-NEXT: ucvtf v7.4s, v7.4s
+; CHECK-FP16-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-FP16-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-NEXT: ushll v19.2d, v4.2s, #0
+; CHECK-FP16-NEXT: ushll v20.2d, v6.2s, #0
+; CHECK-FP16-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-NEXT: ushll v21.2d, v3.2s, #0
+; CHECK-FP16-NEXT: ushll v22.2d, v5.2s, #0
+; CHECK-FP16-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-FP16-NEXT: ucvtf v18.2d, v17.2d
+; CHECK-FP16-NEXT: ucvtf v17.2d, v0.2d
+; CHECK-FP16-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-FP16-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-FP16-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-FP16-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-FP16-NEXT: ucvtf v22.2d, v22.2d
+; CHECK-FP16-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-FP16-NEXT: fcvtxn v0.2s, v18.2d
+; CHECK-FP16-NEXT: ushll v18.2d, v1.2s, #0
+; CHECK-FP16-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-FP16-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-FP16-NEXT: ushll v17.2d, v7.2s, #0
+; CHECK-FP16-NEXT: fcvtxn2 v16.4s, v2.2d
+; CHECK-FP16-NEXT: ushll2 v2.2d, v7.4s, #0
+; CHECK-FP16-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-NEXT: fcvtxn v7.2s, v21.2d
+; CHECK-FP16-NEXT: fcvtxn2 v19.4s, v4.2d
+; CHECK-FP16-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-NEXT: fcvtxn2 v20.4s, v6.2d
+; CHECK-FP16-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-FP16-NEXT: fcvtxn v18.2s, v18.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v17.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v7.4s
+; CHECK-FP16-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-NEXT: bfcvtn v3.4h, v20.4s
+; CHECK-FP16-NEXT: fcvtxn v6.2s, v17.2d
+; CHECK-FP16-NEXT: ucvtf v17.2d, v2.2d
+; CHECK-FP16-NEXT: fcvtxn2 v18.4s, v1.2d
+; CHECK-FP16-NEXT: bfcvtn v1.4h, v16.4s
+; CHECK-FP16-NEXT: bfcvtn v2.4h, v19.4s
+; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v17.2d
+; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v6.4s
; CHECK-FP16-NEXT: ret
entry:
%c = uitofp <32 x i32> %a to <32 x bfloat>
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
index 3e7f8424c1334..783bec0684e55 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,SVE
+; RUN: llc -mattr=+sve2,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,SVE
; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING-SVE
target triple = "aarch64-unknown-linux-gnu"
@@ -463,7 +463,8 @@ define <vscale x 2 x bfloat> @sitofp_nxv2i32_to_nxv2bf16(<vscale x 2 x i32> %a)
; CHECK-LABEL: sitofp_nxv2i32_to_nxv2bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: scvtf z0.d, p0/m, z0.s
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: ret
%res = sitofp <vscale x 2 x i32> %a to <vscale x 2 x bfloat>
@@ -474,7 +475,21 @@ define <vscale x 2 x bfloat> @sitofp_nxv2i64_to_nxv2bf16(<vscale x 2 x i64> %a)
; CHECK-LABEL: sitofp_nxv2i64_to_nxv2bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: abs z1.d, p0/m, z0.d
+; CHECK-NEXT: and z0.d, z0.d, #0x8000000000000000
+; CHECK-NEXT: lsr z2.d, z1.d, #53
+; CHECK-NEXT: movprfx z3, z1
+; CHECK-NEXT: and z3.d, z3.d, #0xfff
+; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; CHECK-NEXT: mov z2.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: and z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: cmpne p2.d, p1/z, z3.d, #0
+; CHECK-NEXT: scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: orr z0.d, z1.d, z0.d
+; CHECK-NEXT: mov z1.d, p2/z, #1 // =0x1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: ret
%res = sitofp <vscale x 2 x i64> %a to <vscale x 2 x bfloat>
@@ -524,9 +539,16 @@ define <vscale x 4 x bfloat> @sitofp_nxv4i16_to_nxv4bf16(<vscale x 4 x i16> %a)
define <vscale x 4 x bfloat> @sitofp_nxv4i32_to_nxv4bf16(<vscale x 4 x i32> %a) {
; CHECK-LABEL: sitofp_nxv4i32_to_nxv4bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: sunpkhi z1.d, z0.s
+; CHECK-NEXT: sunpklo z0.d, z0.s
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: scvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
; CHECK-NEXT: ret
%res = sitofp <vscale x 4 x i32> %a to <vscale x 4 x bfloat>
ret <vscale x 4 x bfloat> %res
@@ -536,8 +558,35 @@ define <vscale x 4 x bfloat> @sitofp_nxv4i64_to_nxv4bf16(<vscale x 4 x i64> %a)
; CHECK-LABEL: sitofp_nxv4i64_to_nxv4bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: scvtf z1.s, p0/m, z1.d
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: abs z2.d, p0/m, z1.d
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: abs z3.d, p0/m, z0.d
+; CHECK-NEXT: and z1.d, z1.d, #0x8000000000000000
+; CHECK-NEXT: and z0.d, z0.d, #0x8000000000000000
+; CHECK-NEXT: lsr z4.d, z2.d, #53
+; CHECK-NEXT: lsr z5.d, z3.d, #53
+; CHECK-NEXT: movprfx z6, z3
+; CHECK-NEXT: and z6.d, z6.d, #0xfff
+; CHECK-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-NEXT: cmpne p2.d, p0/z, z5.d, #0
+; CHECK-NEXT: mov z4.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: movprfx z5, z2
+; CHECK-NEXT: and z5.d, z5.d, #0xfff
+; CHECK-NEXT: and z2.d, p1/m, z2.d, z4.d
+; CHECK-NEXT: and z3.d, p2/m, z3.d, z4.d
+; CHECK-NEXT: cmpne p3.d, p1/z, z5.d, #0
+; CHECK-NEXT: cmpne p1.d, p2/z, z6.d, #0
+; CHECK-NEXT: scvtf z2.d, p0/m, z2.d
+; CHECK-NEXT: scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT: mov z4.d, p3/z, #1 // =0x1
+; CHECK-NEXT: orr z1.d, z2.d, z1.d
+; CHECK-NEXT: orr z0.d, z3.d, z0.d
+; CHECK-NEXT: mov z2.d, p1/z, #1 // =0x1
+; CHECK-NEXT: orr z1.d, z1.d, z4.d
+; CHECK-NEXT: orr z0.d, z0.d, z2.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
@@ -599,11 +648,25 @@ define <vscale x 8 x bfloat> @sitofp_nxv8i16_to_nxv8bf16(<vscale x 8 x i16> %a)
define <vscale x 8 x bfloat> @sitofp_nxv8i32_to_nxv8bf16(<vscale x 8 x i32> %a) {
; CHECK-LABEL: sitofp_nxv8i32_to_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: scvtf z1.s, p0/m, z1.s
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: sunpkhi z2.d, z1.s
+; CHECK-NEXT: sunpklo z1.d, z1.s
+; CHECK-NEXT: sunpkhi z3.d, z0.s
+; CHECK-NEXT: sunpklo z0.d, z0.s
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: scvtf z2.d, p0/m, z2.d
+; CHECK-NEXT: scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT: scvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z3.s, p0/m, z3.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-NEXT: bfcvt z2.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z3.h, p0/m, z3.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-NEXT: uzp1 z1.s, z1.s, z2.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z3.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
%res = sitofp <vscale x 8 x i32> %a to <vscale x 8 x bfloat>
@@ -613,11 +676,70 @@ define <vscale x 8 x bfloat> @sitofp_nxv8i32_to_nxv8bf16(<vscale x 8 x i32> %a)
define <vscale x 8 x bfloat> @sitofp_nxv8i64_to_nxv8bf16(<vscale x 8 x i64> %a) {
; CHECK-LABEL: sitofp_nxv8i64_to_nxv8bf16:
; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: scvtf z3.s, p0/m, z3.d
-; CHECK-NEXT: scvtf z2.s, p0/m, z2.d
-; CHECK-NEXT: scvtf z1.s, p0/m, z1.d
-; CHECK-NEXT: scvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: movprfx z4, z3
+; CHECK-NEXT: abs z4.d, p0/m, z3.d
+; CHECK-NEXT: movprfx z5, z2
+; CHECK-NEXT: abs z5.d, p0/m, z2.d
+; CHECK-NEXT: movprfx z6, z1
+; CHECK-NEXT: abs z6.d, p0/m, z1.d
+; CHECK-NEXT: movprfx z7, z0
+; CHECK-NEXT: abs z7.d, p0/m, z0.d
+; CHECK-NEXT: and z3.d, z3.d, #0x8000000000000000
+; CHECK-NEXT: and z2.d, z2.d, #0x8000000000000000
+; CHECK-NEXT: and z1.d, z1.d, #0x8000000000000000
+; CHECK-NEXT: and z0.d, z0.d, #0x8000000000000000
+; CHECK-NEXT: lsr z24.d, z4.d, #53
+; CHECK-NEXT: lsr z25.d, z5.d, #53
+; CHECK-NEXT: lsr z26.d, z6.d, #53
+; CHECK-NEXT: lsr z27.d, z7.d, #53
+; CHECK-NEXT: cmpne p2.d, p0/z, z24.d, #0
+; CHECK-NEXT: cmpne p3.d, p0/z, z25.d, #0
+; CHECK-NEXT: cmpne p4.d, p0/z, z26.d, #0
+; CHECK-NEXT: cmpne p1.d, p0/z, z27.d, #0
+; CHECK-NEXT: mov z24.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: movprfx z25, z4
+; CHECK-NEXT: and z25.d, z25.d, #0xfff
+; CHECK-NEXT: movprfx z26, z5
+; CHECK-NEXT: and z26.d, z26.d, #0xfff
+; CHECK-NEXT: movprfx z27, z6
+; CHECK-NEXT: and z27.d, z27.d, #0xfff
+; CHECK-NEXT: and z4.d, p2/m, z4.d, z24.d
+; CHECK-NEXT: cmpne p2.d, p2/z, z25.d, #0
+; CHECK-NEXT: and z5.d, p3/m, z5.d, z24.d
+; CHECK-NEXT: movprfx z25, z7
+; CHECK-NEXT: and z25.d, z25.d, #0xfff
+; CHECK-NEXT: and z6.d, p4/m, z6.d, z24.d
+; CHECK-NEXT: and z7.d, p1/m, z7.d, z24.d
+; CHECK-NEXT: cmpne p3.d, p3/z, z26.d, #0
+; CHECK-NEXT: cmpne p4.d, p4/z, z27.d, #0
+; CHECK-NEXT: scvtf z4.d, p0/m, z4.d
+; CHECK-NEXT: scvtf z5.d, p0/m, z5.d
+; CHECK-NEXT: cmpne p1.d, p1/z, z25.d, #0
+; CHECK-NEXT: scvtf z6.d, p0/m, z6.d
+; CHECK-NEXT: scvtf z7.d, p0/m, z7.d
+; CHECK-NEXT: mov z24.d, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z25.d, p3/z, #1 // =0x1
+; CHECK-NEXT: mov z26.d, p4/z, #1 // =0x1
+; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: orr z3.d, z4.d, z3.d
+; CHECK-NEXT: orr z2.d, z5.d, z2.d
+; CHECK-NEXT: orr z1.d, z6.d, z1.d
+; CHECK-NEXT: orr z0.d, z7.d, z0.d
+; CHECK-NEXT: mov z4.d, p1/z, #1 // =0x1
+; CHECK-NEXT: orr z3.d, z3.d, z24.d
+; CHECK-NEXT: orr z2.d, z2.d, z25.d
+; CHECK-NEXT: orr z1.d, z1.d, z26.d
+; CHECK-NEXT: orr z0.d, z0.d, z4.d
+; CHECK-NEXT: fcvtx z3.s, p0/m, z3.d
+; CHECK-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z3.h, p0/m, z3.s
; CHECK-NEXT: bfcvt z2.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
@@ -625,6 +747,8 @@ define <vscale x 8 x bfloat> @sitofp_nxv8i64_to_nxv8bf16(<vscale x 8 x i64> %a)
; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s
; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%res = sitofp <vscale x 8 x i64> %a to <vscale x 8 x bfloat>
ret <vscale x 8 x bfloat> %res
@@ -675,7 +799,8 @@ define <vscale x 2 x bfloat> @uitofp_nxv2i32_to_nxv2bf16(<vscale x 2 x i32> %a)
; CHECK-LABEL: uitofp_nxv2i32_to_nxv2bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.s
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: ret
%res = uitofp <vscale x 2 x i32> %a to <vscale x 2 x bfloat>
@@ -685,8 +810,18 @@ define <vscale x 2 x bfloat> @uitofp_nxv2i32_to_nxv2bf16(<vscale x 2 x i32> %a)
define <vscale x 2 x bfloat> @uitofp_nxv2i64_to_nxv2bf16(<vscale x 2 x i64> %a) {
; CHECK-LABEL: uitofp_nxv2i64_to_nxv2bf16:
; CHECK: // %bb.0:
+; CHECK-NEXT: lsr z1.d, z0.d, #53
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: and z2.d, z2.d, #0xfff
+; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0
+; CHECK-NEXT: mov z1.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: and z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: cmpne p2.d, p1/z, z2.d, #0
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z1.d, p2/z, #1 // =0x1
+; CHECK-NEXT: orr z0.d, z0.d, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: ret
%res = uitofp <vscale x 2 x i64> %a to <vscale x 2 x bfloat>
@@ -736,9 +871,16 @@ define <vscale x 4 x bfloat> @uitofp_nxv4i16_to_nxv4bf16(<vscale x 4 x i16> %a)
define <vscale x 4 x bfloat> @uitofp_nxv4i32_to_nxv4bf16(<vscale x 4 x i32> %a) {
; CHECK-LABEL: uitofp_nxv4i32_to_nxv4bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: uunpkhi z1.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
; CHECK-NEXT: ret
%res = uitofp <vscale x 4 x i32> %a to <vscale x 4 x bfloat>
ret <vscale x 4 x bfloat> %res
@@ -747,9 +889,28 @@ define <vscale x 4 x bfloat> @uitofp_nxv4i32_to_nxv4bf16(<vscale x 4 x i32> %a)
define <vscale x 4 x bfloat> @uitofp_nxv4i64_to_nxv4bf16(<vscale x 4 x i64> %a) {
; CHECK-LABEL: uitofp_nxv4i64_to_nxv4bf16:
; CHECK: // %bb.0:
+; CHECK-NEXT: lsr z2.d, z1.d, #53
+; CHECK-NEXT: lsr z3.d, z0.d, #53
+; CHECK-NEXT: movprfx z4, z0
+; CHECK-NEXT: and z4.d, z4.d, #0xfff
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; CHECK-NEXT: cmpne p2.d, p0/z, z3.d, #0
+; CHECK-NEXT: mov z2.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: movprfx z3, z1
+; CHECK-NEXT: and z3.d, z3.d, #0xfff
+; CHECK-NEXT: and z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: cmpne p3.d, p1/z, z3.d, #0
+; CHECK-NEXT: and z0.d, p2/m, z0.d, z2.d
+; CHECK-NEXT: cmpne p1.d, p2/z, z4.d, #0
+; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z2.d, p3/z, #1 // =0x1
+; CHECK-NEXT: mov z3.d, p1/z, #1 // =0x1
+; CHECK-NEXT: orr z1.d, z1.d, z2.d
+; CHECK-NEXT: orr z0.d, z0.d, z3.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
@@ -810,11 +971,25 @@ define <vscale x 8 x bfloat> @uitofp_nxv8i16_to_nxv8bf16(<vscale x 8 x i16> %a)
define <vscale x 8 x bfloat> @uitofp_nxv8i32_to_nxv8bf16(<vscale x 8 x i32> %a) {
; CHECK-LABEL: uitofp_nxv8i32_to_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: ucvtf z1.s, p0/m, z1.s
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s
+; CHECK-NEXT: uunpkhi z2.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: uunpkhi z3.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ucvtf z2.d, p0/m, z2.d
+; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z3.s, p0/m, z3.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-NEXT: bfcvt z2.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z3.h, p0/m, z3.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-NEXT: uzp1 z1.s, z1.s, z2.s
+; CHECK-NEXT: uzp1 z0.s, z0.s, z3.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
%res = uitofp <vscale x 8 x i32> %a to <vscale x 8 x bfloat>
@@ -824,11 +999,54 @@ define <vscale x 8 x bfloat> @uitofp_nxv8i32_to_nxv8bf16(<vscale x 8 x i32> %a)
define <vscale x 8 x bfloat> @uitofp_nxv8i64_to_nxv8bf16(<vscale x 8 x i64> %a) {
; CHECK-LABEL: uitofp_nxv8i64_to_nxv8bf16:
; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: lsr z4.d, z3.d, #53
+; CHECK-NEXT: lsr z5.d, z2.d, #53
+; CHECK-NEXT: lsr z6.d, z1.d, #53
+; CHECK-NEXT: lsr z7.d, z0.d, #53
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ucvtf z3.s, p0/m, z3.d
-; CHECK-NEXT: ucvtf z2.s, p0/m, z2.d
-; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d
-; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d
+; CHECK-NEXT: mov z24.d, #-4096 // =0xfffffffffffff000
+; CHECK-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-NEXT: cmpne p2.d, p0/z, z5.d, #0
+; CHECK-NEXT: movprfx z4, z3
+; CHECK-NEXT: and z4.d, z4.d, #0xfff
+; CHECK-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; CHECK-NEXT: cmpne p4.d, p0/z, z7.d, #0
+; CHECK-NEXT: movprfx z5, z2
+; CHECK-NEXT: and z5.d, z5.d, #0xfff
+; CHECK-NEXT: movprfx z6, z1
+; CHECK-NEXT: and z6.d, z6.d, #0xfff
+; CHECK-NEXT: movprfx z7, z0
+; CHECK-NEXT: and z7.d, z7.d, #0xfff
+; CHECK-NEXT: and z3.d, p1/m, z3.d, z24.d
+; CHECK-NEXT: cmpne p1.d, p1/z, z4.d, #0
+; CHECK-NEXT: and z2.d, p2/m, z2.d, z24.d
+; CHECK-NEXT: cmpne p2.d, p2/z, z5.d, #0
+; CHECK-NEXT: and z1.d, p3/m, z1.d, z24.d
+; CHECK-NEXT: cmpne p3.d, p3/z, z6.d, #0
+; CHECK-NEXT: and z0.d, p4/m, z0.d, z24.d
+; CHECK-NEXT: cmpne p4.d, p4/z, z7.d, #0
+; CHECK-NEXT: ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT: ucvtf z2.d, p0/m, z2.d
+; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT: mov z4.d, p1/z, #1 // =0x1
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z5.d, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z6.d, p3/z, #1 // =0x1
+; CHECK-NEXT: mov z7.d, p4/z, #1 // =0x1
+; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: orr z3.d, z3.d, z4.d
+; CHECK-NEXT: orr z2.d, z2.d, z5.d
+; CHECK-NEXT: orr z1.d, z1.d, z6.d
+; CHECK-NEXT: orr z0.d, z0.d, z7.d
+; CHECK-NEXT: fcvtx z3.s, p0/m, z3.d
+; CHECK-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT: fcvtx z0.s, p0/m, z0.d
; CHECK-NEXT: bfcvt z3.h, p0/m, z3.s
; CHECK-NEXT: bfcvt z2.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
@@ -836,6 +1054,8 @@ define <vscale x 8 x bfloat> @uitofp_nxv8i64_to_nxv8bf16(<vscale x 8 x i64> %a)
; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s
; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%res = uitofp <vscale x 8 x i64> %a to <vscale x 8 x bfloat>
ret <vscale x 8 x bfloat> %res
>From c95679d07210ece07092bc13f36e8ce8bcf619d8 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Fri, 17 Jul 2026 10:16:55 +0100
Subject: [PATCH 2/2] [AArch64][GlobalISel] BF16 sitofp.
This adds lowering for bf16 s/uitofp. It needs to be careful not to
double-round the result and uses lower to perform the correct action based on
the size of the input. This does not yet handle i64 which are more involved.
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 31 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 6 +-
.../test/CodeGen/AArch64/bf16-instructions.ll | 250 +-
.../CodeGen/AArch64/bf16-v4-instructions.ll | 354 +-
.../CodeGen/AArch64/bf16-v8-instructions.ll | 1166 +++-
llvm/test/CodeGen/AArch64/itofp-bf16.ll | 5758 ++++++++++++-----
7 files changed, 5256 insertions(+), 2313 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 19e15b1ae0efb..3fff5f8ec9d38 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -8459,6 +8459,27 @@ loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
return LegalizerHelper::Legalized;
}
+static LegalizerHelper::LegalizeResult
+lowerBF16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
+ LLT SrcTy, MachineIRBuilder &MIRBuilder) {
+ // TODOD: Use max bits from known-bits
+ if (SrcTy.getScalarSizeInBits() <= 24) {
+ LLT CvtTy = SrcTy.changeElementType(LLT::float32());
+ auto Cvt =
+ MIRBuilder.buildInstr(MI.getOpcode(), {CvtTy}, {Src}, MI.getFlags());
+ MIRBuilder.buildFPTrunc(Dst, Cvt, MI.getFlags());
+ } else if (SrcTy.getScalarSizeInBits() <= 53) {
+ LLT CvtTy = SrcTy.changeElementType(LLT::float64());
+ auto Cvt =
+ MIRBuilder.buildInstr(MI.getOpcode(), {CvtTy}, {Src}, MI.getFlags());
+ MIRBuilder.buildFPTrunc(Dst, Cvt, MI.getFlags());
+ } else
+ return LegalizerHelper::UnableToLegalize;
+
+ MI.eraseFromParent();
+ return LegalizerHelper::Legalized;
+}
+
LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
@@ -8470,9 +8491,12 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
return Legalized;
}
- if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+ if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarSizeInBits() == 64)
return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+ if (DstTy.getScalarType().isBFloat16())
+ return lowerBF16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
if (SrcTy != LLT::scalar(64))
return UnableToLegalize;
@@ -8504,9 +8528,12 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
return Legalized;
}
- if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+ if (DstTy.getScalarType().isFloat16() && SrcTy.getScalarSizeInBits() == 64)
return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+ if (DstTy.getScalarType().isBFloat16())
+ return lowerBF16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
if (SrcTy != I64)
return UnableToLegalize;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 17307f64fad7c..2f1239dc78019 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5421,7 +5421,7 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
assert(!(IsStrict && VT.isScalableVector()) &&
"Unimplemented SVE support for ISD:::STRICT_INT_TO_FP!");
- // NOTE: i1->bf16 does not require promotion to f32.
+ // NOTE: i1->float does not require promotion to f32.
if (VT.isScalableVector() && InVT.getVectorElementType() == MVT::i1) {
SDValue FalseVal = DAG.getConstantFP(0.0, DL, VT);
SDValue TrueVal = IsSigned ? DAG.getConstantFP(-1.0, DL, VT)
@@ -8452,7 +8452,7 @@ SDValue AArch64TargetLowering::LowerCLMUL(SDValue Op, SelectionDAG &DAG) const {
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
SelectionDAG &DAG) const {
LLVM_DEBUG(dbgs() << "Custom lowering: ");
- LLVM_DEBUG(Op.dump());
+ LLVM_DEBUG(Op.dump(&DAG));
switch (Op.getOpcode()) {
default:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 94ca171c0b207..a2103baf4bb1d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1020,14 +1020,14 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
{v2f64, v2i64}})
.legalFor(HasFP16,
{{f16, i32}, {f16, i64}, {v4f16, v4i16}, {v8f16, v8i16}})
- .unsupportedIf([&](const LegalityQuery &Query) {
- return Query.Types[0].getScalarType().isBFloat16();
- })
.scalarizeIf(scalarOrEltWiderThan(1, 64), 1)
.scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
.moreElementsToNextPow2(1)
.widenScalarOrEltToNextPow2OrMinSize(1)
.minScalar(1, f32)
+ .lowerIf([](const LegalityQuery &Query) {
+ return Query.Types[0].getScalarType().isBFloat16();
+ })
.lowerIf([](const LegalityQuery &Query) {
return Query.Types[1].isVector() &&
Query.Types[1].getScalarSizeInBits() == 64 &&
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index 9cd31ca7cc99f..0f4d1abd94fa7 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -4,21 +4,13 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_uitofp_i32
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_uitofp_i64
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double_fast
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_uitofp_i32
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_uitofp_i64
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double_fast
@@ -1680,19 +1672,19 @@ define i64 @test_fptoui_sat_i64(bfloat %a) {
}
define bfloat @test_uitofp_i32(i32 %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ucvtf d0, w0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: fcvtxn s0, d0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ucvtf d0, w0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: fcvtxn s0, d0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i32:
; CHECK-BF16: // %bb.0:
@@ -1700,6 +1692,23 @@ define bfloat @test_uitofp_i32(i32 %a) #0 {
; CHECK-BF16-NEXT: fcvtxn s0, d0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ucvtf d0, w0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: fcvtxn s0, d0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = uitofp i32 %a to bfloat
ret bfloat %r
}
@@ -1750,19 +1759,19 @@ define bfloat @test_uitofp_i64(i64 %a) #0 {
}
define bfloat @test_sitofp_i32(i32 %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: scvtf d0, w0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: fcvtxn s0, d0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: scvtf d0, w0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: fcvtxn s0, d0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i32:
; CHECK-BF16: // %bb.0:
@@ -1770,6 +1779,23 @@ define bfloat @test_sitofp_i32(i32 %a) #0 {
; CHECK-BF16-NEXT: fcvtxn s0, d0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: scvtf d0, w0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: fcvtxn s0, d0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = sitofp i32 %a to bfloat
ret bfloat %r
}
@@ -1828,29 +1854,29 @@ define bfloat @test_sitofp_i64(i64 %a) #0 {
}
define bfloat @test_uitofp_i32_fadd(i32 %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i32_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ucvtf d1, w0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fcvtxn s1, d1
-; CHECK-CVT-NEXT: fmov w9, s1
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i32_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ucvtf d1, w0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fcvtxn s1, d1
+; CHECK-CVT-SD-NEXT: fmov w9, s1
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i32_fadd:
; CHECK-BF16: // %bb.0:
@@ -1863,35 +1889,65 @@ define bfloat @test_uitofp_i32_fadd(i32 %a, bfloat %b) #0 {
; CHECK-BF16-NEXT: fadd s0, s0, s1
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i32_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ucvtf d1, w0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fcvtxn s1, d1
+; CHECK-CVT-GI-NEXT: fmov w9, s1
+; CHECK-CVT-GI-NEXT: fcmp s1, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w10, w11, w10
+; CHECK-CVT-GI-NEXT: csel w9, w9, w10, vs
+; CHECK-CVT-GI-NEXT: lsr w9, w9, #16
+; CHECK-CVT-GI-NEXT: fmov s1, w9
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%c = uitofp i32 %a to bfloat
%r = fadd bfloat %b, %c
ret bfloat %r
}
define bfloat @test_sitofp_i32_fadd(i32 %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i32_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: scvtf d1, w0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fcvtxn s1, d1
-; CHECK-CVT-NEXT: fmov w9, s1
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i32_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: scvtf d1, w0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fcvtxn s1, d1
+; CHECK-CVT-SD-NEXT: fmov w9, s1
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i32_fadd:
; CHECK-BF16: // %bb.0:
@@ -1904,6 +1960,36 @@ define bfloat @test_sitofp_i32_fadd(i32 %a, bfloat %b) #0 {
; CHECK-BF16-NEXT: fadd s0, s0, s1
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i32_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: scvtf d1, w0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fcvtxn s1, d1
+; CHECK-CVT-GI-NEXT: fmov w9, s1
+; CHECK-CVT-GI-NEXT: fcmp s1, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w10, w11, w10
+; CHECK-CVT-GI-NEXT: csel w9, w9, w10, vs
+; CHECK-CVT-GI-NEXT: lsr w9, w9, #16
+; CHECK-CVT-GI-NEXT: fmov s1, w9
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%c = sitofp i32 %a to bfloat
%r = fadd bfloat %b, %c
ret bfloat %r
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index 211353482a8bb..607eb65d84991 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -4,22 +4,10 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_sitofp_i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i16
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_sitofp_i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i16
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
define <4 x bfloat> @test_build(<4 x bfloat> %a) {
@@ -1572,44 +1560,73 @@ define <4 x i64> @test_fptoui_sat_i64(<4 x bfloat> %a) {
}
define <4 x bfloat> @test_sitofp_i8(<4 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-CVT-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-CVT-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sitofp_i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-BF16-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-BF16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sitofp_i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BF16-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BF16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-CVT-GI-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-CVT-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sitofp_i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-BF16-GI-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-BF16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = sitofp <4 x i8> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
define <4 x bfloat> @test_sitofp_i16(<4 x i16> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i16:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i16:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i16:
; CHECK-BF16: // %bb.0:
@@ -1617,30 +1634,48 @@ define <4 x bfloat> @test_sitofp_i16(<4 x i16> %a) #0 {
; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i16:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%1 = sitofp <4 x i16> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
define <4 x bfloat> @test_sitofp_i32(<4 x i32> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll v1.2d, v0.2s, #0
-; CHECK-CVT-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: scvtf v1.2d, v1.2d
-; CHECK-CVT-NEXT: scvtf v0.2d, v0.2d
-; CHECK-CVT-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-CVT-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-CVT-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-CVT-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-CVT-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-CVT-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sitofp_i32:
; CHECK-BF16: // %bb.0:
@@ -1652,6 +1687,28 @@ define <4 x bfloat> @test_sitofp_i32(<4 x i32> %a) #0 {
; CHECK-BF16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%1 = sitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
@@ -1702,42 +1759,71 @@ define <4 x bfloat> @test_sitofp_i64(<4 x i64> %a) #0 {
}
define <4 x bfloat> @test_uitofp_i8(<4 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_uitofp_i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-BF16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_uitofp_i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-BF16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: movi v1.2d, #0x0000ff000000ff
+; CHECK-CVT-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_uitofp_i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: movi v1.2d, #0x0000ff000000ff
+; CHECK-BF16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BF16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = uitofp <4 x i8> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
define <4 x bfloat> @test_uitofp_i16(<4 x i16> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i16:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i16:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i16:
; CHECK-BF16: // %bb.0:
@@ -1745,30 +1831,48 @@ define <4 x bfloat> @test_uitofp_i16(<4 x i16> %a) #0 {
; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i16:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%1 = uitofp <4 x i16> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
define <4 x bfloat> @test_uitofp_i32(<4 x i32> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll v1.2d, v0.2s, #0
-; CHECK-CVT-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-CVT-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-CVT-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-CVT-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-CVT-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-CVT-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-CVT-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-CVT-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-CVT-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_uitofp_i32:
; CHECK-BF16: // %bb.0:
@@ -1780,6 +1884,28 @@ define <4 x bfloat> @test_uitofp_i32(<4 x i32> %a) #0 {
; CHECK-BF16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%1 = uitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %1
}
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 6ec3c71c4d6dd..5c4c7f6f662b5 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -5,26 +5,10 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_sitofp_i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_v16i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i16
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_v16i8
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_sitofp_i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_v16i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i16
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_v16i8
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_sitofp_i64
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
define <8 x bfloat> @test_build(<8 x bfloat> %a) {
@@ -3663,177 +3647,446 @@ define <8 x i64> @test_fptoui_sat_i64(<8 x bfloat> %a) {
}
define <8 x bfloat> @test_sitofp_i8(<8 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: scvtf v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: scvtf v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sitofp_i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-BF16-NEXT: sshll v1.4s, v0.4h, #0
-; CHECK-BF16-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: scvtf v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sitofp_i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BF16-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_sitofp_i8:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BF16SVE-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sitofp_i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BF16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: scvtf v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = sitofp <8 x i8> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
define <16 x bfloat> @test_sitofp_v16i8(<16 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_v16i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll2 v2.8h, v0.16b, #0
-; CHECK-CVT-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v7.4s, #127, msl #8
-; CHECK-CVT-NEXT: sshll v3.4s, v2.4h, #0
-; CHECK-CVT-NEXT: sshll v4.4s, v0.4h, #0
-; CHECK-CVT-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: scvtf v3.4s, v3.4s
-; CHECK-CVT-NEXT: scvtf v4.4s, v4.4s
-; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: scvtf v6.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v0.4s, v4.4s, #16
-; CHECK-CVT-NEXT: ushr v16.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-CVT-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v16.16b, v16.16b, v1.16b
-; CHECK-CVT-NEXT: and v17.16b, v17.16b, v1.16b
-; CHECK-CVT-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-CVT-NEXT: addhn v1.4h, v3.4s, v5.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v4.4s, v0.4s
-; CHECK-CVT-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-CVT-NEXT: add v4.4s, v17.4s, v7.4s
-; CHECK-CVT-NEXT: addhn2 v1.8h, v2.4s, v3.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v6.4s, v4.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_v16i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll2 v2.8h, v0.16b, #0
+; CHECK-CVT-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: sshll v3.4s, v2.4h, #0
+; CHECK-CVT-SD-NEXT: sshll v4.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-CVT-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-CVT-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-CVT-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: scvtf v6.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v4.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-CVT-SD-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v16.16b, v16.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v17.16b, v17.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v5.4s, v5.4s, v7.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-CVT-SD-NEXT: addhn v1.4h, v3.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v4.4s, v0.4s
+; CHECK-CVT-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-CVT-SD-NEXT: add v4.4s, v17.4s, v7.4s
+; CHECK-CVT-SD-NEXT: addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sitofp_v16i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: sshll2 v1.8h, v0.16b, #0
-; CHECK-BF16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-BF16-NEXT: sshll v2.4s, v1.4h, #0
-; CHECK-BF16-NEXT: sshll v3.4s, v0.4h, #0
-; CHECK-BF16-NEXT: sshll2 v4.4s, v1.8h, #0
-; CHECK-BF16-NEXT: sshll2 v5.4s, v0.8h, #0
-; CHECK-BF16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-BF16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-BF16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-BF16-NEXT: scvtf v2.4s, v4.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v3.4s
-; CHECK-BF16-NEXT: scvtf v3.4s, v5.4s
-; CHECK-BF16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sitofp_v16i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-BF16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BF16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-BF16-SD-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: sshll2 v4.4s, v1.8h, #0
+; CHECK-BF16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-BF16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-BF16-SD-NEXT: scvtf v2.4s, v4.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-BF16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_sitofp_v16i8:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-BF16SVE-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BF16SVE-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-BF16SVE-SD-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v4.4s, v1.8h, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: scvtf v2.4s, v4.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_v16i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-CVT-GI-NEXT: movi v6.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: sshll v3.4s, v2.4h, #0
+; CHECK-CVT-GI-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-CVT-GI-NEXT: sshll v4.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: scvtf v4.4s, v4.4s
+; CHECK-CVT-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v7.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v18.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v20.4s, v4.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v21.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v19.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v23.4s, v4.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: and v7.16b, v7.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v18.16b, v18.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v20.16b, v20.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v6.16b, v21.16b, v6.16b
+; CHECK-CVT-GI-NEXT: fcmeq v21.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: fcmeq v24.4s, v4.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v4.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: add v5.4s, v16.4s, v7.4s
+; CHECK-CVT-GI-NEXT: add v7.4s, v19.4s, v18.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-CVT-GI-NEXT: add v16.4s, v23.4s, v20.4s
+; CHECK-CVT-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-CVT-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-CVT-GI-NEXT: bif v3.16b, v5.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mov v5.16b, v19.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v4.16b, v16.16b, v18.16b
+; CHECK-CVT-GI-NEXT: bsl v5.16b, v0.16b, v1.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v2.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v4.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-CVT-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sitofp_v16i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-BF16-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-BF16-GI-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-BF16-GI-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-BF16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-BF16-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-BF16-GI-NEXT: scvtf v4.4s, v1.4s
+; CHECK-BF16-GI-NEXT: scvtf v5.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = sitofp <16 x i8> %a to <16 x bfloat>
ret <16 x bfloat> %1
}
define <8 x bfloat> @test_sitofp_i16(<8 x i16> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i16:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: scvtf v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i16:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: scvtf v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sitofp_i16:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: sshll v1.4s, v0.4h, #0
-; CHECK-BF16-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: scvtf v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sitofp_i16:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_sitofp_i16:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i16:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sitofp_i16:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: scvtf v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = sitofp <8 x i16> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
define <8 x bfloat> @test_sitofp_i32(<8 x i32> %a) #0 {
-; CHECK-CVT-LABEL: test_sitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sshll v2.2d, v1.2s, #0
-; CHECK-CVT-NEXT: sshll v3.2d, v0.2s, #0
-; CHECK-CVT-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-CVT-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-CVT-NEXT: scvtf v2.2d, v2.2d
-; CHECK-CVT-NEXT: scvtf v3.2d, v3.2d
-; CHECK-CVT-NEXT: scvtf v1.2d, v1.2d
-; CHECK-CVT-NEXT: scvtf v0.2d, v0.2d
-; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-CVT-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-CVT-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-NEXT: and v4.16b, v4.16b, v0.16b
-; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b
-; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s
-; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v4.4s, v6.4s, v4.4s
-; CHECK-CVT-NEXT: fcmeq v6.4s, v3.4s, v3.4s
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-CVT-NEXT: mov v1.16b, v5.16b
-; CHECK-CVT-NEXT: bsl v1.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: bif v0.16b, v3.16b, v6.16b
-; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: sshll v2.2d, v1.2s, #0
+; CHECK-CVT-SD-NEXT: sshll v3.2d, v0.2s, #0
+; CHECK-CVT-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-CVT-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-CVT-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-CVT-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-CVT-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-SD-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-SD-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-CVT-SD-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: mov v1.16b, v5.16b
+; CHECK-CVT-SD-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-CVT-SD-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sitofp_i32:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: sshll v2.2d, v0.2s, #0
-; CHECK-BF16-NEXT: sshll v3.2d, v1.2s, #0
-; CHECK-BF16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-BF16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-BF16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-BF16-NEXT: scvtf v3.2d, v3.2d
-; CHECK-BF16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-BF16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-BF16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-BF16-NEXT: fcvtxn2 v2.4s, v0.2d
-; CHECK-BF16-NEXT: scvtf v0.2d, v1.2d
-; CHECK-BF16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sitofp_i32:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-BF16-SD-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-BF16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-BF16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-BF16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-BF16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-BF16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-BF16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-BF16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_sitofp_i32:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-BF16SVE-SD-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-BF16SVE-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-BF16SVE-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-BF16SVE-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-BF16SVE-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16SVE-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-CVT-GI-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-CVT-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-CVT-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-CVT-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: mvn v1.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bsl v1.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sitofp_i32:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-BF16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-BF16-GI-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-BF16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-BF16-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-BF16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-BF16-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-BF16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-BF16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = sitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
@@ -3915,177 +4168,446 @@ define <8 x bfloat> @test_sitofp_i64(<8 x i64> %a) #0 {
}
define <8 x bfloat> @test_uitofp_i8(<8 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: ucvtf v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ucvtf v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_uitofp_i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-BF16-NEXT: ushll v1.4s, v0.4h, #0
-; CHECK-BF16-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: ucvtf v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_uitofp_i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BF16-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_uitofp_i8:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BF16SVE-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_uitofp_i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BF16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: ucvtf v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = uitofp <8 x i8> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
define <16 x bfloat> @test_uitofp_v16i8(<16 x i8> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_v16i8:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll2 v2.8h, v0.16b, #0
-; CHECK-CVT-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v7.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushll v3.4s, v2.4h, #0
-; CHECK-CVT-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-CVT-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-CVT-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: ucvtf v6.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v0.4s, v4.4s, #16
-; CHECK-CVT-NEXT: ushr v16.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-CVT-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v16.16b, v16.16b, v1.16b
-; CHECK-CVT-NEXT: and v17.16b, v17.16b, v1.16b
-; CHECK-CVT-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-CVT-NEXT: addhn v1.4h, v3.4s, v5.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v4.4s, v0.4s
-; CHECK-CVT-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-CVT-NEXT: add v4.4s, v17.4s, v7.4s
-; CHECK-CVT-NEXT: addhn2 v1.8h, v2.4s, v3.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v6.4s, v4.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_v16i8:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-CVT-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ushll v3.4s, v2.4h, #0
+; CHECK-CVT-SD-NEXT: ushll v4.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-CVT-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-CVT-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-CVT-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ucvtf v6.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v4.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-CVT-SD-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v16.16b, v16.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v17.16b, v17.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v5.4s, v5.4s, v7.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-CVT-SD-NEXT: addhn v1.4h, v3.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v4.4s, v0.4s
+; CHECK-CVT-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-CVT-SD-NEXT: add v4.4s, v17.4s, v7.4s
+; CHECK-CVT-SD-NEXT: addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_uitofp_v16i8:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ushll2 v1.8h, v0.16b, #0
-; CHECK-BF16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-BF16-NEXT: ushll v2.4s, v1.4h, #0
-; CHECK-BF16-NEXT: ushll v3.4s, v0.4h, #0
-; CHECK-BF16-NEXT: ushll2 v4.4s, v1.8h, #0
-; CHECK-BF16-NEXT: ushll2 v5.4s, v0.8h, #0
-; CHECK-BF16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-BF16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-BF16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-BF16-NEXT: ucvtf v2.4s, v4.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v3.4s
-; CHECK-BF16-NEXT: ucvtf v3.4s, v5.4s
-; CHECK-BF16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_uitofp_v16i8:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-BF16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BF16-SD-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-BF16-SD-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: ushll2 v4.4s, v1.8h, #0
+; CHECK-BF16-SD-NEXT: ushll2 v5.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-BF16-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-BF16-SD-NEXT: ucvtf v2.4s, v4.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-BF16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_uitofp_v16i8:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-BF16SVE-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BF16SVE-SD-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-BF16SVE-SD-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v4.4s, v1.8h, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v5.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: ucvtf v2.4s, v4.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_v16i8:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-CVT-GI-NEXT: movi v6.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ushll v3.4s, v2.4h, #0
+; CHECK-CVT-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-CVT-GI-NEXT: ushll v4.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v7.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v18.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v20.4s, v4.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v21.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v19.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v23.4s, v4.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: and v7.16b, v7.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v18.16b, v18.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v20.16b, v20.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v6.16b, v21.16b, v6.16b
+; CHECK-CVT-GI-NEXT: fcmeq v21.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: fcmeq v24.4s, v4.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v4.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: add v5.4s, v16.4s, v7.4s
+; CHECK-CVT-GI-NEXT: add v7.4s, v19.4s, v18.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-CVT-GI-NEXT: add v16.4s, v23.4s, v20.4s
+; CHECK-CVT-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-CVT-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-CVT-GI-NEXT: bif v3.16b, v5.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mov v5.16b, v19.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v4.16b, v16.16b, v18.16b
+; CHECK-CVT-GI-NEXT: bsl v5.16b, v0.16b, v1.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v2.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v4.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-CVT-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_uitofp_v16i8:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-BF16-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-BF16-GI-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-BF16-GI-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-BF16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-BF16-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-BF16-GI-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-BF16-GI-NEXT: ucvtf v5.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = uitofp <16 x i8> %a to <16 x bfloat>
ret <16 x bfloat> %1
}
define <8 x bfloat> @test_uitofp_i16(<8 x i16> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i16:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT: ucvtf v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i16:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-CVT-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ucvtf v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_uitofp_i16:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ushll v1.4s, v0.4h, #0
-; CHECK-BF16-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: ucvtf v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_uitofp_i16:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-BF16-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_uitofp_i16:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-BF16SVE-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i16:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_uitofp_i16:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BF16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-BF16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: ucvtf v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = uitofp <8 x i16> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
define <8 x bfloat> @test_uitofp_i32(<8 x i32> %a) #0 {
-; CHECK-CVT-LABEL: test_uitofp_i32:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: ushll v2.2d, v1.2s, #0
-; CHECK-CVT-NEXT: ushll v3.2d, v0.2s, #0
-; CHECK-CVT-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-CVT-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-CVT-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-CVT-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-CVT-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-CVT-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-CVT-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-CVT-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8
-; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-NEXT: and v4.16b, v4.16b, v0.16b
-; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b
-; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s
-; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-CVT-NEXT: add v4.4s, v6.4s, v4.4s
-; CHECK-CVT-NEXT: fcmeq v6.4s, v3.4s, v3.4s
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-CVT-NEXT: mov v1.16b, v5.16b
-; CHECK-CVT-NEXT: bsl v1.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: bif v0.16b, v3.16b, v6.16b
-; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_uitofp_i32:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: ushll v2.2d, v1.2s, #0
+; CHECK-CVT-SD-NEXT: ushll v3.2d, v0.2s, #0
+; CHECK-CVT-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-CVT-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-CVT-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-CVT-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-CVT-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-CVT-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-SD-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-SD-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-CVT-SD-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-CVT-SD-NEXT: mov v1.16b, v5.16b
+; CHECK-CVT-SD-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-CVT-SD-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_uitofp_i32:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: ushll v2.2d, v0.2s, #0
-; CHECK-BF16-NEXT: ushll v3.2d, v1.2s, #0
-; CHECK-BF16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-BF16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-BF16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-BF16-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-BF16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-BF16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-BF16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-BF16-NEXT: fcvtxn2 v2.4s, v0.2d
-; CHECK-BF16-NEXT: ucvtf v0.2d, v1.2d
-; CHECK-BF16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_uitofp_i32:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-BF16-SD-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-BF16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-BF16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-BF16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-BF16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-BF16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-BF16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-SD-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-BF16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_uitofp_i32:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-BF16SVE-SD-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-BF16SVE-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-BF16SVE-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-BF16SVE-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-BF16SVE-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16SVE-SD-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-BF16SVE-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_uitofp_i32:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-CVT-GI-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-CVT-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-CVT-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-CVT-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: mvn v1.16b, v17.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bsl v1.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_uitofp_i32:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-BF16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-BF16-GI-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-BF16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-BF16-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-BF16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-BF16-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-BF16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-BF16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-BF16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-BF16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-BF16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-BF16-GI-NEXT: ret
%1 = uitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %1
}
diff --git a/llvm/test/CodeGen/AArch64/itofp-bf16.ll b/llvm/test/CodeGen/AArch64/itofp-bf16.ll
index 90904cec47722..876910f195803 100644
--- a/llvm/test/CodeGen/AArch64/itofp-bf16.ll
+++ b/llvm/test/CodeGen/AArch64/itofp-bf16.ll
@@ -6,12 +6,6 @@
; CHECK-NOFP16-GI: warning: Instruction selection used fallback path for stofp_i64_bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i64_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i32_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i32_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i16_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i16_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i8_bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i8_bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i64_v2bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i64_v2bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i64_v3bf16
@@ -24,52 +18,10 @@
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i64_v16bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i64_v32bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i64_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i32_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i32_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i32_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i32_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i32_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i32_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i32_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i32_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i32_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i32_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i32_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i32_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i16_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i16_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i16_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i16_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i16_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i16_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i16_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i16_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i16_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i16_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i16_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i16_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i8_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i8_v2bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i8_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i8_v3bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i8_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i8_v4bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i8_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i8_v8bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i8_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i8_v16bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i8_v32bf16
-; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i8_v32bf16
; CHECK-NOFP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i48_v4f16
;
; CHECK-FP16-GI: warning: Instruction selection used fallback path for stofp_i64_bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i64_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i32_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i32_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i16_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i16_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_i8_bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_i8_bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i64_v2bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i64_v2bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i64_v3bf16
@@ -82,42 +34,6 @@
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i64_v16bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i64_v32bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i64_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i32_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i32_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i32_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i32_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i32_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i32_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i32_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i32_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i32_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i32_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i32_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i32_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i16_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i16_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i16_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i16_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i16_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i16_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i16_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i16_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i16_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i16_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i16_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i16_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v2i8_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v2i8_v2bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v3i8_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i8_v3bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v4i8_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i8_v4bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v8i8_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v8i8_v8bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v16i8_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v16i8_v16bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for stofp_v32i8_v32bf16
-; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v32i8_v32bf16
; CHECK-FP16-GI-NEXT: warning: Instruction selection used fallback path for utofp_v4i48_v4f16
define bfloat @stofp_i64_bf16(i64 %a) {
@@ -221,19 +137,19 @@ entry:
}
define bfloat @stofp_i32_bf16(i32 %a) {
-; CHECK-NOFP16-LABEL: stofp_i32_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: scvtf d0, w0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: fcvtxn s0, d0
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_i32_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: scvtf d0, w0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_i32_bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -241,25 +157,42 @@ define bfloat @stofp_i32_bf16(i32 %a) {
; CHECK-FP16-NEXT: fcvtxn s0, d0
; CHECK-FP16-NEXT: bfcvt h0, s0
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i32_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: scvtf d0, w0
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp i32 %a to bfloat
ret bfloat %c
}
define bfloat @utofp_i32_bf16(i32 %a) {
-; CHECK-NOFP16-LABEL: utofp_i32_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ucvtf d0, w0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: fcvtxn s0, d0
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_i32_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ucvtf d0, w0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_i32_bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -267,110 +200,231 @@ define bfloat @utofp_i32_bf16(i32 %a) {
; CHECK-FP16-NEXT: fcvtxn s0, d0
; CHECK-FP16-NEXT: bfcvt h0, s0
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i32_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ucvtf d0, w0
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp i32 %a to bfloat
ret bfloat %c
}
define bfloat @stofp_i16_bf16(i16 %a) {
-; CHECK-NOFP16-LABEL: stofp_i16_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sxth w9, w0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: scvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_i16_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sxth w9, w0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: scvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_i16_bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sxth w8, w0
-; CHECK-FP16-NEXT: scvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_i16_bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sxth w8, w0
+; CHECK-FP16-SD-NEXT: scvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i16_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sxth w9, w0
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: scvtf d0, w9
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_i16_bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sxth w8, w0
+; CHECK-FP16-GI-NEXT: scvtf d0, w8
+; CHECK-FP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-FP16-GI-NEXT: bfcvt h0, s0
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp i16 %a to bfloat
ret bfloat %c
}
define bfloat @utofp_i16_bf16(i16 %a) {
-; CHECK-NOFP16-LABEL: utofp_i16_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: and w9, w0, #0xffff
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: ucvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_i16_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: and w9, w0, #0xffff
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: ucvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_i16_bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: and w8, w0, #0xffff
-; CHECK-FP16-NEXT: ucvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_i16_bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: and w8, w0, #0xffff
+; CHECK-FP16-SD-NEXT: ucvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i16_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: and w9, w0, #0xffff
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: ucvtf d0, w9
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_i16_bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: and w8, w0, #0xffff
+; CHECK-FP16-GI-NEXT: ucvtf d0, w8
+; CHECK-FP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-FP16-GI-NEXT: bfcvt h0, s0
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp i16 %a to bfloat
ret bfloat %c
}
define bfloat @stofp_i8_bf16(i8 %a) {
-; CHECK-NOFP16-LABEL: stofp_i8_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sxtb w9, w0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: scvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_i8_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sxtb w9, w0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: scvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_i8_bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sxtb w8, w0
-; CHECK-FP16-NEXT: scvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_i8_bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sxtb w8, w0
+; CHECK-FP16-SD-NEXT: scvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i8_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sxtb w9, w0
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: scvtf d0, w9
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_i8_bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sxtb w8, w0
+; CHECK-FP16-GI-NEXT: scvtf d0, w8
+; CHECK-FP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-FP16-GI-NEXT: bfcvt h0, s0
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp i8 %a to bfloat
ret bfloat %c
}
define bfloat @utofp_i8_bf16(i8 %a) {
-; CHECK-NOFP16-LABEL: utofp_i8_bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: and w9, w0, #0xff
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: ucvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w10, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w8, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_i8_bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: and w9, w0, #0xff
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: ucvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w8, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_i8_bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: and w8, w0, #0xff
-; CHECK-FP16-NEXT: ucvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_i8_bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: and w8, w0, #0xff
+; CHECK-FP16-SD-NEXT: ucvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i8_bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: and w9, w0, #0xff
+; CHECK-NOFP16-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-GI-NEXT: ucvtf d0, w9
+; CHECK-NOFP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-NOFP16-GI-NEXT: fmov w9, s0
+; CHECK-NOFP16-GI-NEXT: fcmp s0, #0.0
+; CHECK-NOFP16-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-NOFP16-GI-NEXT: add w8, w9, w8
+; CHECK-NOFP16-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-NOFP16-GI-NEXT: add w8, w8, w10
+; CHECK-NOFP16-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-NOFP16-GI-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-GI-NEXT: fmov s0, w8
+; CHECK-NOFP16-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_i8_bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: and w8, w0, #0xff
+; CHECK-FP16-GI-NEXT: ucvtf d0, w8
+; CHECK-FP16-GI-NEXT: fcvtxn s0, d0
+; CHECK-FP16-GI-NEXT: bfcvt h0, s0
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp i8 %a to bfloat
ret bfloat %c
@@ -1673,103 +1727,157 @@ entry:
}
define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v2i32_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: bit v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i32_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: bit v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v2i32_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-FP16-NEXT: movi d1, #0000000000000000
-; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: fcvtxn s2, d0
-; CHECK-FP16-NEXT: mov d0, v0.d[1]
-; CHECK-FP16-NEXT: bfcvt h2, s2
-; CHECK-FP16-NEXT: fcvtxn s0, d0
-; CHECK-FP16-NEXT: mov v1.h[0], v2.h[0]
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: mov v1.h[1], v0.h[0]
-; CHECK-FP16-NEXT: fmov d0, d1
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v2i32_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: movi d1, #0000000000000000
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: fcvtxn s2, d0
+; CHECK-FP16-SD-NEXT: mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT: bfcvt h2, s2
+; CHECK-FP16-SD-NEXT: fcvtxn s0, d0
+; CHECK-FP16-SD-NEXT: mov v1.h[0], v2.h[0]
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-FP16-SD-NEXT: fmov d0, d1
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i32_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i32_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i32> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v2i32_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v1.16b, v3.16b, v1.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: bit v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i32_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: bit v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v2i32_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-FP16-NEXT: movi d1, #0000000000000000
-; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: fcvtxn s2, d0
-; CHECK-FP16-NEXT: mov d0, v0.d[1]
-; CHECK-FP16-NEXT: bfcvt h2, s2
-; CHECK-FP16-NEXT: fcvtxn s0, d0
-; CHECK-FP16-NEXT: mov v1.h[0], v2.h[0]
-; CHECK-FP16-NEXT: bfcvt h0, s0
-; CHECK-FP16-NEXT: mov v1.h[1], v0.h[0]
-; CHECK-FP16-NEXT: fmov d0, d1
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v2i32_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: movi d1, #0000000000000000
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: fcvtxn s2, d0
+; CHECK-FP16-SD-NEXT: mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT: bfcvt h2, s2
+; CHECK-FP16-SD-NEXT: fcvtxn s0, d0
+; CHECK-FP16-SD-NEXT: mov v1.h[0], v2.h[0]
+; CHECK-FP16-SD-NEXT: bfcvt h0, s0
+; CHECK-FP16-SD-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-FP16-SD-NEXT: fmov d0, d1
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i32_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i32_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i32> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v3i32_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v1.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i32_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v3i32_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -1781,31 +1889,53 @@ define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i32_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <3 x i32> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v3i32_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v1.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i32_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v3i32_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -1817,31 +1947,53 @@ define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i32_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <3 x i32> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v4i32_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v1.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i32_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v4i32_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -1853,31 +2005,53 @@ define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i32_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v4i32_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v1.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: ushr v3.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-NOFP16-NEXT: and v0.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v3.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-NOFP16-NEXT: shrn v0.4h, v0.4s, #16
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i32_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v4i32_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -1889,749 +2063,1535 @@ define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-FP16-NEXT: fcvtxn2 v1.4s, v0.2d
; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i32_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <4 x i32> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v8i32_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v2.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: sshll v3.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: add v6.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: and v4.16b, v4.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v5.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v5.4s, v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v4.4s, v6.4s, v4.4s
-; CHECK-NOFP16-NEXT: fcmeq v6.4s, v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: mov v1.16b, v5.16b
-; CHECK-NOFP16-NEXT: bsl v1.16b, v4.16b, v2.16b
-; CHECK-NOFP16-NEXT: bif v0.16b, v3.16b, v6.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i32_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: mov v1.16b, v5.16b
+; CHECK-NOFP16-SD-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v8i32_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v2.2d, v0.2s, #0
-; CHECK-FP16-NEXT: sshll v3.2d, v1.2s, #0
-; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v0.2d
-; CHECK-FP16-NEXT: scvtf v0.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v8i32_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i32_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bsl v1.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i32_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v8i32_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v2.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: ushll v3.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-NOFP16-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v4.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: add v6.4s, v2.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: and v4.16b, v4.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v5.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v5.4s, v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v4.4s, v6.4s, v4.4s
-; CHECK-NOFP16-NEXT: fcmeq v6.4s, v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: mov v1.16b, v5.16b
-; CHECK-NOFP16-NEXT: bsl v1.16b, v4.16b, v2.16b
-; CHECK-NOFP16-NEXT: bif v0.16b, v3.16b, v6.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i32_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v5.4s, v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: mov v1.16b, v5.16b
+; CHECK-NOFP16-SD-NEXT: bsl v1.16b, v4.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: bif v0.16b, v3.16b, v6.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v8i32_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v2.2d, v0.2s, #0
-; CHECK-FP16-NEXT: ushll v3.2d, v1.2s, #0
-; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v3.2s, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v0.2d
-; CHECK-FP16-NEXT: ucvtf v0.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn2 v3.4s, v0.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v8i32_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v3.4s, v0.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i32_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: mvn v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bsl v1.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i32_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v3.2s, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v2.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v3.4s, v1.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <8 x i32> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v16i32_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v5.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: sshll v4.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: sshll v6.2d, v3.2s, #0
-; CHECK-NOFP16-NEXT: sshll v7.2d, v2.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v3.2d, v3.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v2.2d, v2.4s, #0
-; CHECK-NOFP16-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NOFP16-NEXT: scvtf v7.2d, v7.2d
-; CHECK-NOFP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-NEXT: fcvtxn v5.2s, v5.2d
-; CHECK-NOFP16-NEXT: fcvtxn v4.2s, v4.2d
-; CHECK-NOFP16-NEXT: fcvtxn v6.2s, v6.2d
-; CHECK-NOFP16-NEXT: fcvtxn v7.2s, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v5.4s, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v4.4s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v6.4s, v3.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v7.4s, v2.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v2.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: add v19.4s, v5.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v18.4s, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v20.4s, v6.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v7.4s, v1.4s
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v17.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v17.4s, v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: orr v4.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v3.4s, v19.4s, v3.4s
-; CHECK-NOFP16-NEXT: fcmeq v19.4s, v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
-; CHECK-NOFP16-NEXT: fcmeq v18.4s, v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: fcmeq v1.4s, v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v20.4s, v16.4s
-; CHECK-NOFP16-NEXT: orr v5.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v6.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v7.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: bif v2.16b, v4.16b, v17.16b
-; CHECK-NOFP16-NEXT: mov v4.16b, v19.16b
-; CHECK-NOFP16-NEXT: bif v3.16b, v5.16b, v18.16b
-; CHECK-NOFP16-NEXT: bsl v1.16b, v0.16b, v7.16b
-; CHECK-NOFP16-NEXT: bsl v4.16b, v16.16b, v6.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v3.8h, v2.8h
-; CHECK-NOFP16-NEXT: uzp2 v1.8h, v1.8h, v4.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i32_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v7.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v5.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v6.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v7.4s, v2.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v19.4s, v5.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v18.4s, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v20.4s, v6.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v2.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v17.4s, v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: orr v4.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v19.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v19.4s, v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v18.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v18.4s, v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v1.4s, v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: orr v5.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v6.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v7.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: bif v2.16b, v4.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: mov v4.16b, v19.16b
+; CHECK-NOFP16-SD-NEXT: bif v3.16b, v5.16b, v18.16b
+; CHECK-NOFP16-SD-NEXT: bsl v1.16b, v0.16b, v7.16b
+; CHECK-NOFP16-SD-NEXT: bsl v4.16b, v16.16b, v6.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v3.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v1.8h, v1.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v16i32_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v4.2d, v0.2s, #0
-; CHECK-FP16-NEXT: sshll v5.2d, v2.2s, #0
-; CHECK-FP16-NEXT: sshll v6.2d, v1.2s, #0
-; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: sshll v7.2d, v3.2s, #0
-; CHECK-FP16-NEXT: sshll2 v2.2d, v2.4s, #0
-; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: sshll2 v3.2d, v3.4s, #0
-; CHECK-FP16-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-NEXT: scvtf v6.2d, v6.2d
-; CHECK-FP16-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: scvtf v7.2d, v7.2d
-; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v4.2s, v4.2d
-; CHECK-FP16-NEXT: fcvtxn v5.2s, v5.2d
-; CHECK-FP16-NEXT: fcvtxn v6.2s, v6.2d
-; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v0.2d
-; CHECK-FP16-NEXT: scvtf v0.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn2 v5.4s, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v2.2s, v7.2d
-; CHECK-FP16-NEXT: scvtf v1.2d, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v0.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
-; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v16i32_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v4.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v5.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: sshll v7.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v2.2s, v7.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v6.4s, v0.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v6.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i32_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v4.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v5.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v6.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v16.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v17.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v4.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v4.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v5.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v2.2s, v6.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v5.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v3.2s, v7.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v0.4s, v16.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v17.2d
+; CHECK-NOFP16-GI-NEXT: movi v17.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v2.4s, v4.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v3.4s, v5.2d
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v18.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v23.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v0.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v1.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v21.4s, v2.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v16.16b, v16.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v18.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v6.4s, v19.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v20.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v21.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v5.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v7.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i32_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v4.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: sshll v5.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v6.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v6.4s, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v6.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <16 x i32> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v16i32_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v5.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: ushll v4.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: ushll v6.2d, v3.2s, #0
-; CHECK-NOFP16-NEXT: ushll v7.2d, v2.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v3.2d, v3.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v2.2d, v2.4s, #0
-; CHECK-NOFP16-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-NOFP16-NEXT: ucvtf v4.2d, v4.2d
-; CHECK-NOFP16-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-NEXT: ucvtf v7.2d, v7.2d
-; CHECK-NOFP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-NOFP16-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-NOFP16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-NOFP16-NEXT: fcvtxn v5.2s, v5.2d
-; CHECK-NOFP16-NEXT: fcvtxn v4.2s, v4.2d
-; CHECK-NOFP16-NEXT: fcvtxn v6.2s, v6.2d
-; CHECK-NOFP16-NEXT: fcvtxn v7.2s, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v5.4s, v0.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v4.4s, v1.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v6.4s, v3.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v7.4s, v2.2d
-; CHECK-NOFP16-NEXT: movi v0.4s, #1
-; CHECK-NOFP16-NEXT: movi v1.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v3.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v2.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: add v19.4s, v5.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v18.4s, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v20.4s, v6.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v7.4s, v1.4s
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v0.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v17.16b, v0.16b
-; CHECK-NOFP16-NEXT: fcmeq v17.4s, v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: orr v4.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v3.4s, v19.4s, v3.4s
-; CHECK-NOFP16-NEXT: fcmeq v19.4s, v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v18.4s, v2.4s
-; CHECK-NOFP16-NEXT: fcmeq v18.4s, v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: fcmeq v1.4s, v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v20.4s, v16.4s
-; CHECK-NOFP16-NEXT: orr v5.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v6.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v7.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: bif v2.16b, v4.16b, v17.16b
-; CHECK-NOFP16-NEXT: mov v4.16b, v19.16b
-; CHECK-NOFP16-NEXT: bif v3.16b, v5.16b, v18.16b
-; CHECK-NOFP16-NEXT: bsl v1.16b, v0.16b, v7.16b
-; CHECK-NOFP16-NEXT: bsl v4.16b, v16.16b, v6.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v3.8h, v2.8h
-; CHECK-NOFP16-NEXT: uzp2 v1.8h, v1.8h, v4.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i32_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v5.4s, v0.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v6.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v7.4s, v2.2d
+; CHECK-NOFP16-SD-NEXT: movi v0.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v19.4s, v5.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v18.4s, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v20.4s, v6.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v2.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v17.4s, v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: orr v4.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v19.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v19.4s, v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v18.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v18.4s, v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v1.4s, v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: orr v5.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v6.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v7.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: bif v2.16b, v4.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: mov v4.16b, v19.16b
+; CHECK-NOFP16-SD-NEXT: bif v3.16b, v5.16b, v18.16b
+; CHECK-NOFP16-SD-NEXT: bsl v1.16b, v0.16b, v7.16b
+; CHECK-NOFP16-SD-NEXT: bsl v4.16b, v16.16b, v6.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v3.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v1.8h, v1.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v16i32_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v4.2d, v0.2s, #0
-; CHECK-FP16-NEXT: ushll v5.2d, v2.2s, #0
-; CHECK-FP16-NEXT: ushll v6.2d, v1.2s, #0
-; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: ushll v7.2d, v3.2s, #0
-; CHECK-FP16-NEXT: ushll2 v2.2d, v2.4s, #0
-; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: ushll2 v3.2d, v3.4s, #0
-; CHECK-FP16-NEXT: ucvtf v4.2d, v4.2d
-; CHECK-FP16-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-FP16-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-NEXT: ucvtf v7.2d, v7.2d
-; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v4.2s, v4.2d
-; CHECK-FP16-NEXT: fcvtxn v5.2s, v5.2d
-; CHECK-FP16-NEXT: fcvtxn v6.2s, v6.2d
-; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v0.2d
-; CHECK-FP16-NEXT: ucvtf v0.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn2 v5.4s, v2.2d
-; CHECK-FP16-NEXT: fcvtxn v2.2s, v7.2d
-; CHECK-FP16-NEXT: ucvtf v1.2d, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v0.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
-; CHECK-FP16-NEXT: fcvtxn2 v2.4s, v1.2d
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v16i32_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v4.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v5.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v6.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: ushll v7.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v2.2s, v7.2d
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v6.4s, v0.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v2.4s, v1.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v6.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i32_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v4.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v5.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v6.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v16.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v17.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v4.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v4.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v1.2s, v5.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v2.2s, v6.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v5.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v3.2s, v7.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v0.4s, v16.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v1.4s, v17.2d
+; CHECK-NOFP16-GI-NEXT: movi v17.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v2.4s, v4.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v3.4s, v5.2d
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v18.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v23.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v0.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v1.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v21.4s, v2.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v16.16b, v16.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v18.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v6.4s, v19.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v20.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v21.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v5.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v7.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i32_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v4.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: ushll v5.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v6.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v4.2s, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v5.2s, v5.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v6.2s, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v7.2s, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v4.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v5.4s, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v6.4s, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v6.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <16 x i32> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) {
-; CHECK-NOFP16-LABEL: stofp_v32i32_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v17.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: sshll v18.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v19.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: sshll v16.2d, v3.2s, #0
-; CHECK-NOFP16-NEXT: sshll v21.2d, v2.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v3.2d, v3.4s, #0
-; CHECK-NOFP16-NEXT: sshll v23.2d, v7.2s, #0
-; CHECK-NOFP16-NEXT: sshll2 v7.2d, v7.4s, #0
-; CHECK-NOFP16-NEXT: scvtf v17.2d, v17.2d
-; CHECK-NOFP16-NEXT: scvtf v18.2d, v18.2d
-; CHECK-NOFP16-NEXT: scvtf v20.2d, v1.2d
-; CHECK-NOFP16-NEXT: scvtf v19.2d, v19.2d
-; CHECK-NOFP16-NEXT: scvtf v16.2d, v16.2d
-; CHECK-NOFP16-NEXT: scvtf v22.2d, v3.2d
-; CHECK-NOFP16-NEXT: scvtf v23.2d, v23.2d
-; CHECK-NOFP16-NEXT: scvtf v7.2d, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v17.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v18.2d
-; CHECK-NOFP16-NEXT: sshll2 v18.2d, v2.4s, #0
-; CHECK-NOFP16-NEXT: scvtf v2.2d, v21.2d
-; CHECK-NOFP16-NEXT: sshll v21.2d, v4.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v16.2d
-; CHECK-NOFP16-NEXT: sshll2 v4.2d, v4.4s, #0
-; CHECK-NOFP16-NEXT: movi v17.4s, #1
-; CHECK-NOFP16-NEXT: movi v16.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v18.2d, v18.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v20.2d
-; CHECK-NOFP16-NEXT: sshll v20.2d, v5.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v19.2d
-; CHECK-NOFP16-NEXT: sshll v19.2d, v6.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-NOFP16-NEXT: scvtf v21.2d, v21.2d
-; CHECK-NOFP16-NEXT: sshll2 v5.2d, v5.4s, #0
-; CHECK-NOFP16-NEXT: sshll2 v6.2d, v6.4s, #0
-; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v22.2d
-; CHECK-NOFP16-NEXT: scvtf v20.2d, v20.2d
-; CHECK-NOFP16-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-NEXT: scvtf v19.2d, v19.2d
-; CHECK-NOFP16-NEXT: ushr v22.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: add v24.4s, v0.4s, v16.4s
-; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v18.2d
-; CHECK-NOFP16-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NOFP16-NEXT: ushr v25.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: ushr v26.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: fcmeq v27.4s, v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: fcvtxn v18.2s, v20.2d
-; CHECK-NOFP16-NEXT: fcvtxn v20.2s, v21.2d
-; CHECK-NOFP16-NEXT: fcvtxn v21.2s, v23.2d
-; CHECK-NOFP16-NEXT: fcvtxn v19.2s, v19.2d
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v17.16b
-; CHECK-NOFP16-NEXT: fcmeq v23.4s, v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: fcvtxn2 v18.4s, v5.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v20.4s, v4.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v21.4s, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v19.4s, v6.2d
-; CHECK-NOFP16-NEXT: add v22.4s, v24.4s, v22.4s
-; CHECK-NOFP16-NEXT: and v24.16b, v25.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v25.4s, v1.4s, v16.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: ushr v26.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: add v4.4s, v3.4s, v16.4s
-; CHECK-NOFP16-NEXT: fcmeq v7.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: ushr v28.4s, v21.4s, #16
-; CHECK-NOFP16-NEXT: add v30.4s, v20.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v31.4s, v21.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v25.4s, v24.4s
-; CHECK-NOFP16-NEXT: and v24.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v25.4s, v2.4s, v16.4s
-; CHECK-NOFP16-NEXT: ushr v26.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: add v4.4s, v4.4s, v5.4s
-; CHECK-NOFP16-NEXT: ushr v5.4s, v20.4s, #16
-; CHECK-NOFP16-NEXT: ushr v29.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: and v28.16b, v28.16b, v17.16b
-; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v24.4s, v25.4s, v24.4s
-; CHECK-NOFP16-NEXT: bit v1.16b, v6.16b, v7.16b
-; CHECK-NOFP16-NEXT: bit v0.16b, v22.16b, v23.16b
-; CHECK-NOFP16-NEXT: and v25.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v26.4s, v18.4s, v16.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v17.16b
-; CHECK-NOFP16-NEXT: and v17.16b, v29.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v16.4s, v19.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v28.4s, v31.4s, v28.4s
-; CHECK-NOFP16-NEXT: fcmeq v31.4s, v21.4s, v21.4s
-; CHECK-NOFP16-NEXT: fcmeq v29.4s, v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v25.4s, v26.4s, v25.4s
-; CHECK-NOFP16-NEXT: fcmeq v26.4s, v18.4s, v18.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v30.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v16.4s, v17.4s
-; CHECK-NOFP16-NEXT: fcmeq v17.4s, v19.4s, v19.4s
-; CHECK-NOFP16-NEXT: fcmeq v30.4s, v20.4s, v20.4s
-; CHECK-NOFP16-NEXT: orr v18.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v20.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v21.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v19.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: bit v3.16b, v4.16b, v27.16b
-; CHECK-NOFP16-NEXT: mov v6.16b, v31.16b
-; CHECK-NOFP16-NEXT: mov v4.16b, v26.16b
-; CHECK-NOFP16-NEXT: bit v2.16b, v24.16b, v29.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v1.8h, v0.8h
-; CHECK-NOFP16-NEXT: mov v7.16b, v17.16b
-; CHECK-NOFP16-NEXT: bif v5.16b, v20.16b, v30.16b
-; CHECK-NOFP16-NEXT: bsl v6.16b, v28.16b, v21.16b
-; CHECK-NOFP16-NEXT: bsl v4.16b, v25.16b, v18.16b
-; CHECK-NOFP16-NEXT: bsl v7.16b, v16.16b, v19.16b
-; CHECK-NOFP16-NEXT: uzp2 v1.8h, v2.8h, v3.8h
-; CHECK-NOFP16-NEXT: uzp2 v2.8h, v5.8h, v4.8h
-; CHECK-NOFP16-NEXT: uzp2 v3.8h, v7.8h, v6.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i32_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v18.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v19.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v21.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v23.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v20.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v22.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v23.2d, v23.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v0.2s, v17.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v18.2d
+; CHECK-NOFP16-SD-NEXT: sshll2 v18.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT: sshll v21.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn v3.2s, v16.2d
+; CHECK-NOFP16-SD-NEXT: sshll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v17.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v16.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v0.4s, v20.2d
+; CHECK-NOFP16-SD-NEXT: sshll v20.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v19.2d
+; CHECK-NOFP16-SD-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v21.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v3.4s, v22.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v24.4s, v0.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v2.4s, v18.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: fcmeq v27.4s, v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: fcvtxn v18.2s, v20.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v20.2s, v21.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v21.2s, v23.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v18.4s, v5.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v21.4s, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v19.4s, v6.2d
+; CHECK-NOFP16-SD-NEXT: add v22.4s, v24.4s, v22.4s
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v25.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v3.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: ushr v28.4s, v21.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v30.4s, v20.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v31.4s, v21.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v25.4s, v24.4s
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v2.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v20.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v29.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v28.16b, v28.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v24.4s, v25.4s, v24.4s
+; CHECK-NOFP16-SD-NEXT: bit v1.16b, v6.16b, v7.16b
+; CHECK-NOFP16-SD-NEXT: bit v0.16b, v22.16b, v23.16b
+; CHECK-NOFP16-SD-NEXT: and v25.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v26.4s, v18.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v5.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: and v17.16b, v29.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v19.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v28.4s, v31.4s, v28.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v31.4s, v21.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v29.4s, v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v26.4s, v25.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v26.4s, v18.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v30.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v17.4s, v19.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v30.4s, v20.4s, v20.4s
+; CHECK-NOFP16-SD-NEXT: orr v18.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v20.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v21.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v19.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: bit v3.16b, v4.16b, v27.16b
+; CHECK-NOFP16-SD-NEXT: mov v6.16b, v31.16b
+; CHECK-NOFP16-SD-NEXT: mov v4.16b, v26.16b
+; CHECK-NOFP16-SD-NEXT: bit v2.16b, v24.16b, v29.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-SD-NEXT: mov v7.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: bif v5.16b, v20.16b, v30.16b
+; CHECK-NOFP16-SD-NEXT: bsl v6.16b, v28.16b, v21.16b
+; CHECK-NOFP16-SD-NEXT: bsl v4.16b, v25.16b, v18.16b
+; CHECK-NOFP16-SD-NEXT: bsl v7.16b, v16.16b, v19.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v2.8h, v5.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v3.8h, v7.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v32i32_v32bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v17.2d, v0.2s, #0
-; CHECK-FP16-NEXT: sshll v16.2d, v2.2s, #0
-; CHECK-FP16-NEXT: sshll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: sshll v19.2d, v4.2s, #0
-; CHECK-FP16-NEXT: sshll v20.2d, v6.2s, #0
-; CHECK-FP16-NEXT: sshll2 v2.2d, v2.4s, #0
-; CHECK-FP16-NEXT: sshll v21.2d, v3.2s, #0
-; CHECK-FP16-NEXT: sshll v22.2d, v5.2s, #0
-; CHECK-FP16-NEXT: sshll2 v4.2d, v4.4s, #0
-; CHECK-FP16-NEXT: scvtf v18.2d, v17.2d
-; CHECK-FP16-NEXT: scvtf v17.2d, v0.2d
-; CHECK-FP16-NEXT: scvtf v16.2d, v16.2d
-; CHECK-FP16-NEXT: scvtf v19.2d, v19.2d
-; CHECK-FP16-NEXT: scvtf v20.2d, v20.2d
-; CHECK-FP16-NEXT: sshll2 v6.2d, v6.4s, #0
-; CHECK-FP16-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: scvtf v21.2d, v21.2d
-; CHECK-FP16-NEXT: scvtf v22.2d, v22.2d
-; CHECK-FP16-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-NEXT: sshll2 v3.2d, v3.4s, #0
-; CHECK-FP16-NEXT: sshll2 v5.2d, v5.4s, #0
-; CHECK-FP16-NEXT: fcvtxn v0.2s, v18.2d
-; CHECK-FP16-NEXT: sshll v18.2d, v1.2s, #0
-; CHECK-FP16-NEXT: fcvtxn v16.2s, v16.2d
-; CHECK-FP16-NEXT: fcvtxn v19.2s, v19.2d
-; CHECK-FP16-NEXT: fcvtxn v20.2s, v20.2d
-; CHECK-FP16-NEXT: scvtf v6.2d, v6.2d
-; CHECK-FP16-NEXT: sshll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-NEXT: scvtf v18.2d, v18.2d
-; CHECK-FP16-NEXT: fcvtxn2 v0.4s, v17.2d
-; CHECK-FP16-NEXT: sshll v17.2d, v7.2s, #0
-; CHECK-FP16-NEXT: fcvtxn2 v16.4s, v2.2d
-; CHECK-FP16-NEXT: sshll2 v2.2d, v7.4s, #0
-; CHECK-FP16-NEXT: scvtf v1.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn v7.2s, v21.2d
-; CHECK-FP16-NEXT: fcvtxn2 v19.4s, v4.2d
-; CHECK-FP16-NEXT: fcvtxn v4.2s, v22.2d
-; CHECK-FP16-NEXT: fcvtxn2 v20.4s, v6.2d
-; CHECK-FP16-NEXT: scvtf v17.2d, v17.2d
-; CHECK-FP16-NEXT: fcvtxn v18.2s, v18.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: fcvtxn2 v7.4s, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v5.2d
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v20.4s
-; CHECK-FP16-NEXT: fcvtxn v6.2s, v17.2d
-; CHECK-FP16-NEXT: scvtf v17.2d, v2.2d
-; CHECK-FP16-NEXT: fcvtxn2 v18.4s, v1.2d
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v19.4s
-; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v17.2d
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v7.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v6.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v32i32_v32bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v17.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: sshll v19.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v20.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT: sshll v21.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v22.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v4.2d, v4.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v18.2d, v17.2d
+; CHECK-FP16-SD-NEXT: scvtf v17.2d, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT: scvtf v19.2d, v19.2d
+; CHECK-FP16-SD-NEXT: scvtf v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT: sshll2 v6.2d, v6.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: scvtf v21.2d, v21.2d
+; CHECK-FP16-SD-NEXT: scvtf v22.2d, v22.2d
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn v0.2s, v18.2d
+; CHECK-FP16-SD-NEXT: sshll v18.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: scvtf v18.2d, v18.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-FP16-SD-NEXT: sshll v17.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn2 v16.4s, v2.2d
+; CHECK-FP16-SD-NEXT: sshll2 v2.2d, v7.4s, #0
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v7.2s, v21.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v19.4s, v4.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v20.4s, v6.2d
+; CHECK-FP16-SD-NEXT: scvtf v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v18.2s, v18.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v3.4h, v20.4s
+; CHECK-FP16-SD-NEXT: fcvtxn v6.2s, v17.2d
+; CHECK-FP16-SD-NEXT: scvtf v17.2d, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v18.4s, v1.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v16.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v2.4h, v19.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v6.4s, v17.2d
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v3.8h, v6.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i32_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v17.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v18.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v20.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll v21.2d, v4.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v22.2d, v4.4s, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v16.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v25.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: sshll v24.2d, v5.2s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v26.2d, v21.2d
+; CHECK-NOFP16-GI-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v18.2d, v6.4s, #0
+; CHECK-NOFP16-GI-NEXT: fcvtxn v19.2s, v16.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v4.2s, v17.2d
+; CHECK-NOFP16-GI-NEXT: sshll v16.2d, v6.2s, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v17.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v2.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT: sshll v6.2d, v7.2s, #0
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT: sshll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v21.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v23.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v20.2d, v22.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v19.4s, v25.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v25.2d, v24.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v27.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v6.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v24.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v18.2s, v26.2d
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-NOFP16-GI-NEXT: scvtf v22.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v16.2s, v5.2d
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v19.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn v7.2s, v25.2d
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v6.4s, v23.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v17.2s, v27.2d
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v18.4s, v20.2d
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v19.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: and v23.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v4.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v7.4s, v21.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v16.4s, v24.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v17.4s, v22.2d
+; CHECK-NOFP16-GI-NEXT: orr v5.16b, v19.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v19.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v27.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v25.4s, v23.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v7.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v27.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: ushr v27.4s, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: orr v24.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: and v22.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v6.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: and v28.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v0.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v18.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v27.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v21.16b, v4.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v4.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v22.4s, v25.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v29.4s, v28.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v16.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v17.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v8.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v6.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v6.4s, v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v7.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: orr v16.16b, v16.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v17.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: mvn v0.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v4.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: mvn v6.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v9.16b, v28.16b
+; CHECK-NOFP16-GI-NEXT: bif v7.16b, v27.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v4.16b, v21.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v31.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mov v18.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bif v16.16b, v25.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v24.16b, v26.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v18.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: shrn v7.4h, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v7.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v32i32_v32bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: sshll v18.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT: sshll v20.2d, v4.2s, #0
+; CHECK-FP16-GI-NEXT: sshll v21.2d, v6.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v17.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v19.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v4.2d, v4.4s, #0
+; CHECK-FP16-GI-NEXT: scvtf v16.2d, v16.2d
+; CHECK-FP16-GI-NEXT: scvtf v18.2d, v18.2d
+; CHECK-FP16-GI-NEXT: scvtf v20.2d, v20.2d
+; CHECK-FP16-GI-NEXT: scvtf v21.2d, v21.2d
+; CHECK-FP16-GI-NEXT: sshll v22.2d, v5.2s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v6.2d, v6.4s, #0
+; CHECK-FP16-GI-NEXT: sshll v23.2d, v7.2s, #0
+; CHECK-FP16-GI-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: scvtf v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: scvtf v19.2d, v19.2d
+; CHECK-FP16-GI-NEXT: scvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-GI-NEXT: scvtf v22.2d, v22.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v18.2s, v18.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-GI-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT: scvtf v23.2d, v23.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v21.2s, v21.2d
+; CHECK-FP16-GI-NEXT: sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v5.2d, v5.4s, #0
+; CHECK-FP16-GI-NEXT: sshll2 v7.2d, v7.4s, #0
+; CHECK-FP16-GI-NEXT: fcvtxn v17.2s, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v16.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v18.4s, v2.2d
+; CHECK-FP16-GI-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-FP16-GI-NEXT: scvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v22.2s, v23.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v21.4s, v6.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v16.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v17.4s, v1.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v18.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v19.4s, v3.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v2.4h, v20.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v3.4h, v21.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v22.4s, v7.2d
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v17.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v19.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v3.8h, v22.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <32 x i32> %a to <32 x bfloat>
ret <32 x bfloat> %c
}
define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) {
-; CHECK-NOFP16-LABEL: utofp_v32i32_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v17.2d, v1.2s, #0
-; CHECK-NOFP16-NEXT: ushll v18.2d, v0.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v19.2d, v0.4s, #0
-; CHECK-NOFP16-NEXT: ushll v16.2d, v3.2s, #0
-; CHECK-NOFP16-NEXT: ushll v21.2d, v2.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v3.2d, v3.4s, #0
-; CHECK-NOFP16-NEXT: ushll v23.2d, v7.2s, #0
-; CHECK-NOFP16-NEXT: ushll2 v7.2d, v7.4s, #0
-; CHECK-NOFP16-NEXT: ucvtf v17.2d, v17.2d
-; CHECK-NOFP16-NEXT: ucvtf v18.2d, v18.2d
-; CHECK-NOFP16-NEXT: ucvtf v20.2d, v1.2d
-; CHECK-NOFP16-NEXT: ucvtf v19.2d, v19.2d
-; CHECK-NOFP16-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-NOFP16-NEXT: ucvtf v22.2d, v3.2d
-; CHECK-NOFP16-NEXT: ucvtf v23.2d, v23.2d
-; CHECK-NOFP16-NEXT: ucvtf v7.2d, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn v0.2s, v17.2d
-; CHECK-NOFP16-NEXT: fcvtxn v1.2s, v18.2d
-; CHECK-NOFP16-NEXT: ushll2 v18.2d, v2.4s, #0
-; CHECK-NOFP16-NEXT: ucvtf v2.2d, v21.2d
-; CHECK-NOFP16-NEXT: ushll v21.2d, v4.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn v3.2s, v16.2d
-; CHECK-NOFP16-NEXT: ushll2 v4.2d, v4.4s, #0
-; CHECK-NOFP16-NEXT: movi v17.4s, #1
-; CHECK-NOFP16-NEXT: movi v16.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v18.2d, v18.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v0.4s, v20.2d
-; CHECK-NOFP16-NEXT: ushll v20.2d, v5.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn2 v1.4s, v19.2d
-; CHECK-NOFP16-NEXT: ushll v19.2d, v6.2s, #0
-; CHECK-NOFP16-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-NOFP16-NEXT: ucvtf v21.2d, v21.2d
-; CHECK-NOFP16-NEXT: ushll2 v5.2d, v5.4s, #0
-; CHECK-NOFP16-NEXT: ushll2 v6.2d, v6.4s, #0
-; CHECK-NOFP16-NEXT: fcvtxn2 v3.4s, v22.2d
-; CHECK-NOFP16-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-NOFP16-NEXT: ucvtf v4.2d, v4.2d
-; CHECK-NOFP16-NEXT: ucvtf v19.2d, v19.2d
-; CHECK-NOFP16-NEXT: ushr v22.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: add v24.4s, v0.4s, v16.4s
-; CHECK-NOFP16-NEXT: fcvtxn2 v2.4s, v18.2d
-; CHECK-NOFP16-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-NOFP16-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-NEXT: ushr v25.4s, v1.4s, #16
-; CHECK-NOFP16-NEXT: ushr v26.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: fcmeq v27.4s, v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: fcvtxn v18.2s, v20.2d
-; CHECK-NOFP16-NEXT: fcvtxn v20.2s, v21.2d
-; CHECK-NOFP16-NEXT: fcvtxn v21.2s, v23.2d
-; CHECK-NOFP16-NEXT: fcvtxn v19.2s, v19.2d
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v17.16b
-; CHECK-NOFP16-NEXT: fcmeq v23.4s, v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: orr v0.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: fcvtxn2 v18.4s, v5.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v20.4s, v4.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v21.4s, v7.2d
-; CHECK-NOFP16-NEXT: fcvtxn2 v19.4s, v6.2d
-; CHECK-NOFP16-NEXT: add v22.4s, v24.4s, v22.4s
-; CHECK-NOFP16-NEXT: and v24.16b, v25.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v25.4s, v1.4s, v16.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: ushr v26.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: add v4.4s, v3.4s, v16.4s
-; CHECK-NOFP16-NEXT: fcmeq v7.4s, v1.4s, v1.4s
-; CHECK-NOFP16-NEXT: orr v1.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: ushr v28.4s, v21.4s, #16
-; CHECK-NOFP16-NEXT: add v30.4s, v20.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v31.4s, v21.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v25.4s, v24.4s
-; CHECK-NOFP16-NEXT: and v24.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v25.4s, v2.4s, v16.4s
-; CHECK-NOFP16-NEXT: ushr v26.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: add v4.4s, v4.4s, v5.4s
-; CHECK-NOFP16-NEXT: ushr v5.4s, v20.4s, #16
-; CHECK-NOFP16-NEXT: ushr v29.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: and v28.16b, v28.16b, v17.16b
-; CHECK-NOFP16-NEXT: orr v3.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v24.4s, v25.4s, v24.4s
-; CHECK-NOFP16-NEXT: bit v1.16b, v6.16b, v7.16b
-; CHECK-NOFP16-NEXT: bit v0.16b, v22.16b, v23.16b
-; CHECK-NOFP16-NEXT: and v25.16b, v26.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v26.4s, v18.4s, v16.4s
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v17.16b
-; CHECK-NOFP16-NEXT: and v17.16b, v29.16b, v17.16b
-; CHECK-NOFP16-NEXT: add v16.4s, v19.4s, v16.4s
-; CHECK-NOFP16-NEXT: add v28.4s, v31.4s, v28.4s
-; CHECK-NOFP16-NEXT: fcmeq v31.4s, v21.4s, v21.4s
-; CHECK-NOFP16-NEXT: fcmeq v29.4s, v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: orr v2.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: add v25.4s, v26.4s, v25.4s
-; CHECK-NOFP16-NEXT: fcmeq v26.4s, v18.4s, v18.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v30.4s, v5.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v16.4s, v17.4s
-; CHECK-NOFP16-NEXT: fcmeq v17.4s, v19.4s, v19.4s
-; CHECK-NOFP16-NEXT: fcmeq v30.4s, v20.4s, v20.4s
-; CHECK-NOFP16-NEXT: orr v18.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v20.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v21.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: orr v19.4s, #64, lsl #16
-; CHECK-NOFP16-NEXT: bit v3.16b, v4.16b, v27.16b
-; CHECK-NOFP16-NEXT: mov v6.16b, v31.16b
-; CHECK-NOFP16-NEXT: mov v4.16b, v26.16b
-; CHECK-NOFP16-NEXT: bit v2.16b, v24.16b, v29.16b
-; CHECK-NOFP16-NEXT: uzp2 v0.8h, v1.8h, v0.8h
-; CHECK-NOFP16-NEXT: mov v7.16b, v17.16b
-; CHECK-NOFP16-NEXT: bif v5.16b, v20.16b, v30.16b
-; CHECK-NOFP16-NEXT: bsl v6.16b, v28.16b, v21.16b
-; CHECK-NOFP16-NEXT: bsl v4.16b, v25.16b, v18.16b
-; CHECK-NOFP16-NEXT: bsl v7.16b, v16.16b, v19.16b
-; CHECK-NOFP16-NEXT: uzp2 v1.8h, v2.8h, v3.8h
-; CHECK-NOFP16-NEXT: uzp2 v2.8h, v5.8h, v4.8h
-; CHECK-NOFP16-NEXT: uzp2 v3.8h, v7.8h, v6.8h
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i32_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v18.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v19.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v21.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v23.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v20.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v22.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v23.2d, v23.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v0.2s, v17.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v1.2s, v18.2d
+; CHECK-NOFP16-SD-NEXT: ushll2 v18.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT: ushll v21.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn v3.2s, v16.2d
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-SD-NEXT: movi v17.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v16.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v0.4s, v20.2d
+; CHECK-NOFP16-SD-NEXT: ushll v20.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v1.4s, v19.2d
+; CHECK-NOFP16-SD-NEXT: ushll v19.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v3.4s, v22.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v24.4s, v0.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v2.4s, v18.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: fcmeq v27.4s, v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: fcvtxn v18.2s, v20.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v20.2s, v21.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v21.2s, v23.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: orr v0.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v18.4s, v5.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v21.4s, v7.2d
+; CHECK-NOFP16-SD-NEXT: fcvtxn2 v19.4s, v6.2d
+; CHECK-NOFP16-SD-NEXT: add v22.4s, v24.4s, v22.4s
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v25.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v3.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: ushr v28.4s, v21.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v30.4s, v20.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v31.4s, v21.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v25.4s, v24.4s
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v2.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ushr v26.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v20.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v29.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v28.16b, v28.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: orr v3.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v24.4s, v25.4s, v24.4s
+; CHECK-NOFP16-SD-NEXT: bit v1.16b, v6.16b, v7.16b
+; CHECK-NOFP16-SD-NEXT: bit v0.16b, v22.16b, v23.16b
+; CHECK-NOFP16-SD-NEXT: and v25.16b, v26.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v26.4s, v18.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v5.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: and v17.16b, v29.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v19.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: add v28.4s, v31.4s, v28.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v31.4s, v21.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v29.4s, v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: orr v2.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: add v25.4s, v26.4s, v25.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v26.4s, v18.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v30.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v17.4s, v19.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: fcmeq v30.4s, v20.4s, v20.4s
+; CHECK-NOFP16-SD-NEXT: orr v18.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v20.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v21.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: orr v19.4s, #64, lsl #16
+; CHECK-NOFP16-SD-NEXT: bit v3.16b, v4.16b, v27.16b
+; CHECK-NOFP16-SD-NEXT: mov v6.16b, v31.16b
+; CHECK-NOFP16-SD-NEXT: mov v4.16b, v26.16b
+; CHECK-NOFP16-SD-NEXT: bit v2.16b, v24.16b, v29.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-SD-NEXT: mov v7.16b, v17.16b
+; CHECK-NOFP16-SD-NEXT: bif v5.16b, v20.16b, v30.16b
+; CHECK-NOFP16-SD-NEXT: bsl v6.16b, v28.16b, v21.16b
+; CHECK-NOFP16-SD-NEXT: bsl v4.16b, v25.16b, v18.16b
+; CHECK-NOFP16-SD-NEXT: bsl v7.16b, v16.16b, v19.16b
+; CHECK-NOFP16-SD-NEXT: uzp2 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v2.8h, v5.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT: uzp2 v3.8h, v7.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v32i32_v32bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v17.2d, v0.2s, #0
-; CHECK-FP16-NEXT: ushll v16.2d, v2.2s, #0
-; CHECK-FP16-NEXT: ushll2 v0.2d, v0.4s, #0
-; CHECK-FP16-NEXT: ushll v19.2d, v4.2s, #0
-; CHECK-FP16-NEXT: ushll v20.2d, v6.2s, #0
-; CHECK-FP16-NEXT: ushll2 v2.2d, v2.4s, #0
-; CHECK-FP16-NEXT: ushll v21.2d, v3.2s, #0
-; CHECK-FP16-NEXT: ushll v22.2d, v5.2s, #0
-; CHECK-FP16-NEXT: ushll2 v4.2d, v4.4s, #0
-; CHECK-FP16-NEXT: ucvtf v18.2d, v17.2d
-; CHECK-FP16-NEXT: ucvtf v17.2d, v0.2d
-; CHECK-FP16-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-FP16-NEXT: ucvtf v19.2d, v19.2d
-; CHECK-FP16-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-FP16-NEXT: ushll2 v6.2d, v6.4s, #0
-; CHECK-FP16-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-FP16-NEXT: ucvtf v21.2d, v21.2d
-; CHECK-FP16-NEXT: ucvtf v22.2d, v22.2d
-; CHECK-FP16-NEXT: ucvtf v4.2d, v4.2d
-; CHECK-FP16-NEXT: ushll2 v3.2d, v3.4s, #0
-; CHECK-FP16-NEXT: ushll2 v5.2d, v5.4s, #0
-; CHECK-FP16-NEXT: fcvtxn v0.2s, v18.2d
-; CHECK-FP16-NEXT: ushll v18.2d, v1.2s, #0
-; CHECK-FP16-NEXT: fcvtxn v16.2s, v16.2d
-; CHECK-FP16-NEXT: fcvtxn v19.2s, v19.2d
-; CHECK-FP16-NEXT: fcvtxn v20.2s, v20.2d
-; CHECK-FP16-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-NEXT: ushll2 v1.2d, v1.4s, #0
-; CHECK-FP16-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-FP16-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-FP16-NEXT: ucvtf v18.2d, v18.2d
-; CHECK-FP16-NEXT: fcvtxn2 v0.4s, v17.2d
-; CHECK-FP16-NEXT: ushll v17.2d, v7.2s, #0
-; CHECK-FP16-NEXT: fcvtxn2 v16.4s, v2.2d
-; CHECK-FP16-NEXT: ushll2 v2.2d, v7.4s, #0
-; CHECK-FP16-NEXT: ucvtf v1.2d, v1.2d
-; CHECK-FP16-NEXT: fcvtxn v7.2s, v21.2d
-; CHECK-FP16-NEXT: fcvtxn2 v19.4s, v4.2d
-; CHECK-FP16-NEXT: fcvtxn v4.2s, v22.2d
-; CHECK-FP16-NEXT: fcvtxn2 v20.4s, v6.2d
-; CHECK-FP16-NEXT: ucvtf v17.2d, v17.2d
-; CHECK-FP16-NEXT: fcvtxn v18.2s, v18.2d
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: fcvtxn2 v7.4s, v3.2d
-; CHECK-FP16-NEXT: fcvtxn2 v4.4s, v5.2d
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v20.4s
-; CHECK-FP16-NEXT: fcvtxn v6.2s, v17.2d
-; CHECK-FP16-NEXT: ucvtf v17.2d, v2.2d
-; CHECK-FP16-NEXT: fcvtxn2 v18.4s, v1.2d
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v19.4s
-; CHECK-FP16-NEXT: fcvtxn2 v6.4s, v17.2d
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v7.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v6.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v32i32_v32bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT: ushll v19.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v20.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT: ushll v21.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v22.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v18.2d, v17.2d
+; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v0.2d
+; CHECK-FP16-SD-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-FP16-SD-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-FP16-SD-NEXT: ucvtf v22.2d, v22.2d
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn v0.2s, v18.2d
+; CHECK-FP16-SD-NEXT: ushll v18.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: fcvtxn2 v16.4s, v2.2d
+; CHECK-FP16-SD-NEXT: ushll2 v2.2d, v7.4s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v7.2s, v21.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v19.4s, v4.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v20.4s, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT: fcvtxn v18.2s, v18.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v7.4s, v3.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v3.4h, v20.4s
+; CHECK-FP16-SD-NEXT: fcvtxn v6.2s, v17.2d
+; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v2.2d
+; CHECK-FP16-SD-NEXT: fcvtxn2 v18.4s, v1.2d
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v16.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v2.4h, v19.4s
+; CHECK-FP16-SD-NEXT: fcvtxn2 v6.4s, v17.2d
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v7.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v3.8h, v6.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i32_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: ushll v16.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v17.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v18.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v20.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll v21.2d, v4.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v22.2d, v4.4s, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v25.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT: ushll v24.2d, v5.2s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v26.2d, v21.2d
+; CHECK-NOFP16-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v18.2d, v6.4s, #0
+; CHECK-NOFP16-GI-NEXT: fcvtxn v19.2s, v16.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v4.2s, v17.2d
+; CHECK-NOFP16-GI-NEXT: ushll v16.2d, v6.2s, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v17.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v2.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT: ushll v6.2d, v7.2s, #0
+; CHECK-NOFP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT: ushll2 v7.2d, v7.4s, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v21.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v23.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v20.2d, v22.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v19.4s, v25.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v25.2d, v24.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v4.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v27.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v6.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v24.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v18.2s, v26.2d
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v0.4s, v17.2d
+; CHECK-NOFP16-GI-NEXT: ucvtf v22.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v16.2s, v5.2d
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v19.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn v7.2s, v25.2d
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v6.4s, v23.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn v17.2s, v27.2d
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v18.4s, v20.2d
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v19.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: and v23.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v4.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v7.4s, v21.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v16.4s, v24.2d
+; CHECK-NOFP16-GI-NEXT: fcvtxn2 v17.4s, v22.2d
+; CHECK-NOFP16-GI-NEXT: orr v5.16b, v19.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v19.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v27.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v25.4s, v23.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v7.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v27.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: ushr v27.4s, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: orr v24.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: and v22.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v6.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: and v28.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v0.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v18.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v27.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v21.16b, v4.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v4.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v22.4s, v25.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v29.4s, v28.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v16.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v17.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v8.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v6.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v6.4s, v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v7.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: orr v16.16b, v16.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v17.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: mvn v0.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v4.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: mvn v6.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v9.16b, v28.16b
+; CHECK-NOFP16-GI-NEXT: bif v7.16b, v27.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v4.16b, v21.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v31.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mov v18.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bif v16.16b, v25.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v24.16b, v26.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v18.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: shrn v7.4h, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v7.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v32i32_v32bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v16.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT: ushll v18.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT: ushll v20.2d, v4.2s, #0
+; CHECK-FP16-GI-NEXT: ushll v21.2d, v6.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v17.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v19.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-FP16-GI-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-FP16-GI-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-FP16-GI-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-FP16-GI-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-FP16-GI-NEXT: ushll v22.2d, v5.2s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-FP16-GI-NEXT: ushll v23.2d, v7.2s, #0
+; CHECK-FP16-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-FP16-GI-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v16.2s, v16.2d
+; CHECK-FP16-GI-NEXT: ucvtf v22.2d, v22.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v18.2s, v18.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v20.2s, v20.2d
+; CHECK-FP16-GI-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT: ucvtf v23.2d, v23.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v21.2s, v21.2d
+; CHECK-FP16-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-FP16-GI-NEXT: ushll2 v7.2d, v7.4s, #0
+; CHECK-FP16-GI-NEXT: fcvtxn v17.2s, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v16.4s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v19.2s, v19.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v18.4s, v2.2d
+; CHECK-FP16-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v20.4s, v4.2d
+; CHECK-FP16-GI-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v4.2s, v22.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v22.2s, v23.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v21.4s, v6.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v16.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v17.4s, v1.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v18.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v19.4s, v3.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v2.4h, v20.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v4.4s, v5.2d
+; CHECK-FP16-GI-NEXT: bfcvtn v3.4h, v21.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v22.4s, v7.2d
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v17.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v19.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v2.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v3.8h, v22.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <32 x i32> %a to <32 x bfloat>
ret <32 x bfloat> %c
}
define <2 x bfloat> @stofp_v2i16_v2bf16(<2 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v2i16_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i16_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v2i16_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-FP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v2i16_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i16_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i16_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-FP16-GI-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-FP16-GI-NEXT: scvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i16> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <2 x bfloat> @utofp_v2i16_v2bf16(<2 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v2i16_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i16_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v2i16_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-FP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v2i16_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i16_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: movi d1, #0x00ffff0000ffff
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i16_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: movi d1, #0x00ffff0000ffff
+; CHECK-FP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT: ucvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i16> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v3i16_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i16_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v3i16_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -2639,23 +3599,41 @@ define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) {
; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i16_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <3 x i16> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v3i16_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i16_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v3i16_v3bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -2663,23 +3641,41 @@ define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) {
; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i16_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <3 x i16> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v4i16_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i16_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v4i16_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -2687,23 +3683,41 @@ define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) {
; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i16_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <4 x i16> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v4i16_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i16_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v4i16_v4bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -2711,233 +3725,471 @@ define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) {
; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i16_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <4 x i16> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <8 x bfloat> @stofp_v8i16_v8bf16(<8 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v8i16_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v4.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i16_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v8i16_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v1.4s, v0.4h, #0
-; CHECK-FP16-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-FP16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-FP16-NEXT: scvtf v1.4s, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v8i16_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i16_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i16_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT: scvtf v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <8 x i16> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <8 x bfloat> @utofp_v8i16_v8bf16(<8 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v8i16_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v4.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i16_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v8i16_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v1.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-FP16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-FP16-NEXT: ucvtf v1.4s, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v8i16_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i16_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i16_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT: ucvtf v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <8 x i16> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <16 x bfloat> @stofp_v16i16_v16bf16(<16 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v16i16_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v3.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll v4.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: movi v7.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: scvtf v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: scvtf v6.4s, v1.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v17.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v2.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v5.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i16_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v17.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v2.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v5.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v6.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v16i16_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-FP16-NEXT: sshll v3.4s, v1.4h, #0
-; CHECK-FP16-NEXT: sshll2 v4.4s, v0.8h, #0
-; CHECK-FP16-NEXT: sshll2 v5.4s, v1.8h, #0
-; CHECK-FP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-FP16-NEXT: scvtf v2.4s, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v3.4s
-; CHECK-FP16-NEXT: scvtf v3.4s, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v16i16_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v4.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v5.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i16_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v17.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v16.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v18.4s, v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v0.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: and v16.16b, v16.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v1.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v5.16b, v21.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v24.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v6.4s, v18.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v19.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v6.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bit v4.16b, v1.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v0.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i16_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT: scvtf v4.4s, v0.4s
+; CHECK-FP16-GI-NEXT: scvtf v5.4s, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <16 x i16> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <16 x bfloat> @utofp_v16i16_v16bf16(<16 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v16i16_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v3.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll v4.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: movi v7.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: ucvtf v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: ucvtf v6.4s, v1.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v17.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v2.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v5.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i16_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v17.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v2.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v5.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v6.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v16i16_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ushll v3.4s, v1.4h, #0
-; CHECK-FP16-NEXT: ushll2 v4.4s, v0.8h, #0
-; CHECK-FP16-NEXT: ushll2 v5.4s, v1.8h, #0
-; CHECK-FP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-FP16-NEXT: ucvtf v2.4s, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v3.4s
-; CHECK-FP16-NEXT: ucvtf v3.4s, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v16i16_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i16_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v17.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v16.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v23.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v18.4s, v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v0.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v20.4s, v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: and v16.16b, v16.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v1.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v5.16b, v21.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v24.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v6.4s, v18.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v23.16b
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v19.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v6.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bit v4.16b, v1.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v0.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i16_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT: ucvtf v4.4s, v0.4s
+; CHECK-FP16-GI-NEXT: ucvtf v5.4s, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <16 x i16> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) {
-; CHECK-NOFP16-LABEL: stofp_v32i16_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v4.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: sshll v5.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll v6.4s, v2.4h, #0
-; CHECK-NOFP16-NEXT: sshll v7.4s, v3.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-NOFP16-NEXT: movi v16.4s, #1
-; CHECK-NOFP16-NEXT: scvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: scvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: scvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: scvtf v17.4s, v0.4s
-; CHECK-NOFP16-NEXT: scvtf v18.4s, v1.4s
-; CHECK-NOFP16-NEXT: scvtf v19.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v20.4s, v3.4s
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v0.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v2.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v3.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v20.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v23.16b, v23.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v24.16b, v24.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v25.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v2.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v3.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v7.4s, v3.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v22.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v23.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v24.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v7.4s, v16.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v17.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v18.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v19.4s, v6.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v20.4s, v7.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i16_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v16.4s, #1
+; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v17.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v18.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v19.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v20.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: movi v21.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v17.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v23.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v24.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v20.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v2.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v23.16b, v23.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v24.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v25.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v2.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v3.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: addhn v2.4h, v6.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: addhn v3.4h, v7.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v22.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v23.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v24.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v7.4s, v16.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v17.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v18.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v2.8h, v19.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v3.8h, v20.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: stofp_v32i16_v32bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -2966,65 +4218,173 @@ define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) {
; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v18.4s
; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i16_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: sshll v4.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v6.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: sshll v7.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v16.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v22.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v21.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v19.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #1
+; CHECK-NOFP16-GI-NEXT: scvtf v18.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: scvtf v1.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v7.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v17.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v20.4s, v22.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v24.4s, v21.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v22.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v21.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v19.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v19.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v19.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v1.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v20.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v24.16b, v24.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v20.4s, v19.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v7.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v18.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v25.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v27.4s, v24.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v28.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v18.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: ushr v26.4s, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v23.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v6.16b, v22.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v22.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v25.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: orr v23.16b, v21.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v21.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v28.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v26.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v1.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v29.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v16.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v28.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v17.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v8.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v7.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: orr v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v5.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: mvn v1.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: mvn v21.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v22.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v3.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: mov v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v17.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: bif v7.16b, v26.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: bif v16.16b, v28.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: bit v4.16b, v0.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v6.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mov v6.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v23.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v9.16b, v25.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v31.16b, v27.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v18.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v18.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <32 x i16> %a to <32 x bfloat>
ret <32 x bfloat> %c
}
define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) {
-; CHECK-NOFP16-LABEL: utofp_v32i16_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v4.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: ushll v5.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll v6.4s, v2.4h, #0
-; CHECK-NOFP16-NEXT: ushll v7.4s, v3.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-NOFP16-NEXT: movi v16.4s, #1
-; CHECK-NOFP16-NEXT: ucvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: ucvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: ucvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: ucvtf v17.4s, v0.4s
-; CHECK-NOFP16-NEXT: ucvtf v18.4s, v1.4s
-; CHECK-NOFP16-NEXT: ucvtf v19.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v20.4s, v3.4s
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushr v0.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v2.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v3.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v20.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v2.16b, v2.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v23.16b, v23.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v24.16b, v24.16b, v16.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v25.16b, v16.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v2.4s, v2.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v3.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v4.4s, v1.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v6.4s, v2.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v7.4s, v3.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v22.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v23.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v24.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v7.4s, v16.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v17.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v18.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v19.4s, v6.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v20.4s, v7.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i16_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v6.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: movi v16.4s, #1
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v17.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v18.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v19.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v20.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: movi v21.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v17.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v23.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v24.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v20.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v2.16b, v2.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v23.16b, v23.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v24.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v25.16b, v16.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v2.4s, v2.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v3.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: addhn v2.4h, v6.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: addhn v3.4h, v7.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v22.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v23.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v24.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v7.4s, v16.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v17.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v18.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v2.8h, v19.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v3.8h, v20.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-LABEL: utofp_v32i16_v32bf16:
; CHECK-FP16: // %bb.0: // %entry
@@ -3053,588 +4413,1414 @@ define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) {
; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v18.4s
; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i16_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: ushll v4.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v6.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushll v7.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v16.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v22.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v21.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v19.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #1
+; CHECK-NOFP16-GI-NEXT: ucvtf v18.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v7.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v17.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v20.4s, v22.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v24.4s, v21.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v22.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v21.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v6.4s, v19.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v19.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v19.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v1.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v20.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v24.16b, v24.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v20.4s, v19.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v6.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v7.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v18.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v25.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v27.4s, v24.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v28.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v18.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: ushr v26.4s, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v23.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v6.16b, v22.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v22.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v25.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: orr v23.16b, v21.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v21.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v28.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v26.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v1.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v29.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v16.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v28.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v17.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v8.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v7.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: orr v16.16b, v16.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v17.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v5.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: mvn v1.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: mvn v21.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v22.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v3.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: mov v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v17.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: bif v7.16b, v26.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: bif v16.16b, v28.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: bit v4.16b, v0.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v6.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mov v6.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v23.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v9.16b, v25.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v31.16b, v27.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v18.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v18.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <32 x i16> %a to <32 x bfloat>
ret <32 x bfloat> %c
}
define <2 x bfloat> @stofp_v2i8_v2bf16(<2 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v2i8_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT: mov w9, v0.s[1]
-; CHECK-NOFP16-NEXT: fmov w10, s0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: sxtb w10, w10
-; CHECK-NOFP16-NEXT: sxtb w9, w9
-; CHECK-NOFP16-NEXT: scvtf s1, w10
-; CHECK-NOFP16-NEXT: scvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w10, s1
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w12, w10, #16, #1
-; CHECK-NOFP16-NEXT: ubfx w11, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w9, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: add w8, w12, w8
-; CHECK-NOFP16-NEXT: add w9, w11, w9
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: lsr w9, w9, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: fmov s1, w9
-; CHECK-NOFP16-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i8_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT: mov w9, v0.s[1]
+; CHECK-NOFP16-SD-NEXT: fmov w10, s0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: sxtb w10, w10
+; CHECK-NOFP16-SD-NEXT: sxtb w9, w9
+; CHECK-NOFP16-SD-NEXT: scvtf s1, w10
+; CHECK-NOFP16-SD-NEXT: scvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w10, s1
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-NOFP16-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w9, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w12, w8
+; CHECK-NOFP16-SD-NEXT: add w9, w11, w9
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: lsr w9, w9, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: fmov s1, w9
+; CHECK-NOFP16-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v2i8_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT: mov w8, v0.s[1]
-; CHECK-FP16-NEXT: fmov w9, s0
-; CHECK-FP16-NEXT: sxtb w9, w9
-; CHECK-FP16-NEXT: sxtb w8, w8
-; CHECK-FP16-NEXT: scvtf s1, w9
-; CHECK-FP16-NEXT: scvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h2, s0
-; CHECK-FP16-NEXT: bfcvt h0, s1
-; CHECK-FP16-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v2i8_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT: mov w8, v0.s[1]
+; CHECK-FP16-SD-NEXT: fmov w9, s0
+; CHECK-FP16-SD-NEXT: sxtb w9, w9
+; CHECK-FP16-SD-NEXT: sxtb w8, w8
+; CHECK-FP16-SD-NEXT: scvtf s1, w9
+; CHECK-FP16-SD-NEXT: scvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h2, s0
+; CHECK-FP16-SD-NEXT: bfcvt h0, s1
+; CHECK-FP16-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i8_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-GI-NEXT: scvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i8_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-GI-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-FP16-GI-NEXT: scvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i8> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <2 x bfloat> @utofp_v2i8_v2bf16(<2 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v2i8_v2bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-NEXT: mov w9, v0.s[1]
-; CHECK-NOFP16-NEXT: fmov w10, s0
-; CHECK-NOFP16-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-NOFP16-NEXT: and w10, w10, #0xff
-; CHECK-NOFP16-NEXT: and w9, w9, #0xff
-; CHECK-NOFP16-NEXT: ucvtf s1, w10
-; CHECK-NOFP16-NEXT: ucvtf s0, w9
-; CHECK-NOFP16-NEXT: fmov w10, s1
-; CHECK-NOFP16-NEXT: fmov w9, s0
-; CHECK-NOFP16-NEXT: ubfx w12, w10, #16, #1
-; CHECK-NOFP16-NEXT: ubfx w11, w9, #16, #1
-; CHECK-NOFP16-NEXT: add w9, w9, w8
-; CHECK-NOFP16-NEXT: add w8, w10, w8
-; CHECK-NOFP16-NEXT: add w8, w12, w8
-; CHECK-NOFP16-NEXT: add w9, w11, w9
-; CHECK-NOFP16-NEXT: lsr w8, w8, #16
-; CHECK-NOFP16-NEXT: lsr w9, w9, #16
-; CHECK-NOFP16-NEXT: fmov s0, w8
-; CHECK-NOFP16-NEXT: fmov s1, w9
-; CHECK-NOFP16-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-NOFP16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i8_v2bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT: mov w9, v0.s[1]
+; CHECK-NOFP16-SD-NEXT: fmov w10, s0
+; CHECK-NOFP16-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NOFP16-SD-NEXT: and w10, w10, #0xff
+; CHECK-NOFP16-SD-NEXT: and w9, w9, #0xff
+; CHECK-NOFP16-SD-NEXT: ucvtf s1, w10
+; CHECK-NOFP16-SD-NEXT: ucvtf s0, w9
+; CHECK-NOFP16-SD-NEXT: fmov w10, s1
+; CHECK-NOFP16-SD-NEXT: fmov w9, s0
+; CHECK-NOFP16-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-NOFP16-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-NOFP16-SD-NEXT: add w9, w9, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w10, w8
+; CHECK-NOFP16-SD-NEXT: add w8, w12, w8
+; CHECK-NOFP16-SD-NEXT: add w9, w11, w9
+; CHECK-NOFP16-SD-NEXT: lsr w8, w8, #16
+; CHECK-NOFP16-SD-NEXT: lsr w9, w9, #16
+; CHECK-NOFP16-SD-NEXT: fmov s0, w8
+; CHECK-NOFP16-SD-NEXT: fmov s1, w9
+; CHECK-NOFP16-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v2i8_v2bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT: mov w8, v0.s[1]
-; CHECK-FP16-NEXT: fmov w9, s0
-; CHECK-FP16-NEXT: and w9, w9, #0xff
-; CHECK-FP16-NEXT: and w8, w8, #0xff
-; CHECK-FP16-NEXT: ucvtf s1, w9
-; CHECK-FP16-NEXT: ucvtf s0, w8
-; CHECK-FP16-NEXT: bfcvt h2, s0
-; CHECK-FP16-NEXT: bfcvt h0, s1
-; CHECK-FP16-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-FP16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v2i8_v2bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT: mov w8, v0.s[1]
+; CHECK-FP16-SD-NEXT: fmov w9, s0
+; CHECK-FP16-SD-NEXT: and w9, w9, #0xff
+; CHECK-FP16-SD-NEXT: and w8, w8, #0xff
+; CHECK-FP16-SD-NEXT: ucvtf s1, w9
+; CHECK-FP16-SD-NEXT: ucvtf s0, w8
+; CHECK-FP16-SD-NEXT: bfcvt h2, s0
+; CHECK-FP16-SD-NEXT: bfcvt h0, s1
+; CHECK-FP16-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i8_v2bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i8_v2bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-FP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT: ucvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i8> %a to <2 x bfloat>
ret <2 x bfloat> %c
}
define <3 x bfloat> @stofp_v3i8_v3bf16(<3 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v3i8_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: fmov s0, w0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: mov v0.h[1], w1
-; CHECK-NOFP16-NEXT: mov v0.h[2], w2
-; CHECK-NOFP16-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i8_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: fmov s0, w0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT: mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v3i8_v3bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: fmov s0, w0
-; CHECK-FP16-NEXT: mov v0.h[1], w1
-; CHECK-FP16-NEXT: mov v0.h[2], w2
-; CHECK-FP16-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v3i8_v3bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov s0, w0
+; CHECK-FP16-SD-NEXT: mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT: mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i8_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: fmov s0, w0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT: mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT: mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT: mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i8_v3bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: fmov s0, w0
+; CHECK-FP16-GI-NEXT: mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT: mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT: mov v1.h[0], v0.h[2]
+; CHECK-FP16-GI-NEXT: mov v1.h[1], v0.h[3]
+; CHECK-FP16-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <3 x i8> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <3 x bfloat> @utofp_v3i8_v3bf16(<3 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v3i8_v3bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: fmov s0, w0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: mov v0.h[1], w1
-; CHECK-NOFP16-NEXT: mov v0.h[2], w2
-; CHECK-NOFP16-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i8_v3bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: fmov s0, w0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT: mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v3i8_v3bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: fmov s0, w0
-; CHECK-FP16-NEXT: mov v0.h[1], w1
-; CHECK-FP16-NEXT: mov v0.h[2], w2
-; CHECK-FP16-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v3i8_v3bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov s0, w0
+; CHECK-FP16-SD-NEXT: mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT: mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i8_v3bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: fmov s0, w0
+; CHECK-NOFP16-GI-NEXT: movi d1, #0xff00ff00ff00ff
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT: mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT: mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT: mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i8_v3bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: fmov s0, w0
+; CHECK-FP16-GI-NEXT: movi d1, #0xff00ff00ff00ff
+; CHECK-FP16-GI-NEXT: mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT: mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT: mov v1.h[0], v0.h[2]
+; CHECK-FP16-GI-NEXT: mov v1.h[1], v0.h[3]
+; CHECK-FP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <3 x i8> %a to <3 x bfloat>
ret <3 x bfloat> %c
}
define <4 x bfloat> @stofp_v4i8_v4bf16(<4 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v4i8_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i8_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v4i8_v4bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-NEXT: sshll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: scvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v4i8_v4bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i8_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i8_v4bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-FP16-GI-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-FP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <4 x i8> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <4 x bfloat> @utofp_v4i8_v4bf16(<4 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v4i8_v4bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-NOFP16-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-NOFP16-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i8_v4bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v4i8_v4bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ucvtf v0.4s, v0.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v4i8_v4bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i8_v4bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: movi v1.2d, #0x0000ff000000ff
+; CHECK-NOFP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v4.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i8_v4bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: movi v1.2d, #0x0000ff000000ff
+; CHECK-FP16-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <4 x i8> %a to <4 x bfloat>
ret <4 x bfloat> %c
}
define <8 x bfloat> @stofp_v8i8_v8bf16(<8 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v8i8_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v4.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v8i8_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-FP16-NEXT: sshll v1.4s, v0.4h, #0
-; CHECK-FP16-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-FP16-NEXT: scvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-FP16-NEXT: scvtf v1.4s, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v8i8_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i8_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT: scvtf v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <8 x i8> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <8 x bfloat> @utofp_v8i8_v8bf16(<8 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v8i8_v8bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v4.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v3.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-NOFP16-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v2.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v3.4s, v1.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: add v1.4s, v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v2.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v3.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v8i8_v8bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-FP16-NEXT: ushll v1.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-FP16-NEXT: ucvtf v1.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-FP16-NEXT: ucvtf v1.4s, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v8i8_v8bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-NOFP16-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: bit v2.16b, v0.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i8_v8bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: ucvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT: ucvtf v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <8 x i8> %a to <8 x bfloat>
ret <8 x bfloat> %c
}
define <16 x bfloat> @stofp_v16i8_v16bf16(<16 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v16i8_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll2 v2.8h, v0.16b, #0
-; CHECK-NOFP16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v7.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: sshll v3.4s, v2.4h, #0
-; CHECK-NOFP16-NEXT: sshll v4.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: scvtf v6.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: ushr v0.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v17.16b, v17.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v3.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v4.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v17.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v2.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v6.4s, v4.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll2 v2.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: sshll v3.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v17.16b, v17.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v5.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v4.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v17.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v16i8_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll2 v1.8h, v0.16b, #0
-; CHECK-FP16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-FP16-NEXT: sshll v2.4s, v1.4h, #0
-; CHECK-FP16-NEXT: sshll v3.4s, v0.4h, #0
-; CHECK-FP16-NEXT: sshll2 v4.4s, v1.8h, #0
-; CHECK-FP16-NEXT: sshll2 v5.4s, v0.8h, #0
-; CHECK-FP16-NEXT: scvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: scvtf v3.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: scvtf v2.4s, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v3.4s
-; CHECK-FP16-NEXT: scvtf v3.4s, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v16i8_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v4.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: sshll v3.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v4.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v18.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v20.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v18.16b, v18.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v20.16b, v20.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v21.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v24.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v4.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v16.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v19.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v23.4s, v20.4s
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v5.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v7.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v4.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i8_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: scvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT: scvtf v4.4s, v1.4s
+; CHECK-FP16-GI-NEXT: scvtf v5.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <16 x i8> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <16 x bfloat> @utofp_v16i8_v16bf16(<16 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v16i8_v16bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll2 v2.8h, v0.16b, #0
-; CHECK-NOFP16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: movi v1.4s, #1
-; CHECK-NOFP16-NEXT: movi v7.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushll v3.4s, v2.4h, #0
-; CHECK-NOFP16-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-NOFP16-NEXT: ucvtf v6.4s, v0.4s
-; CHECK-NOFP16-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-NOFP16-NEXT: ushr v0.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v16.4s, v2.4s, #16
-; CHECK-NOFP16-NEXT: ushr v17.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: and v5.16b, v5.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v16.16b, v16.16b, v1.16b
-; CHECK-NOFP16-NEXT: and v17.16b, v17.16b, v1.16b
-; CHECK-NOFP16-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v3.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v4.4s, v0.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v16.4s, v7.4s
-; CHECK-NOFP16-NEXT: add v4.4s, v17.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v2.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v6.4s, v4.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v1.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v7.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v16.4s, v2.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v17.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v16.16b, v16.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: and v17.16b, v17.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v5.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v4.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v16.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: add v4.4s, v17.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v6.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v16i8_v16bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll2 v1.8h, v0.16b, #0
-; CHECK-FP16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-FP16-NEXT: ushll v2.4s, v1.4h, #0
-; CHECK-FP16-NEXT: ushll v3.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ushll2 v4.4s, v1.8h, #0
-; CHECK-FP16-NEXT: ushll2 v5.4s, v0.8h, #0
-; CHECK-FP16-NEXT: ucvtf v2.4s, v2.4s
-; CHECK-FP16-NEXT: ucvtf v3.4s, v3.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v2.4s
-; CHECK-FP16-NEXT: ucvtf v2.4s, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v3.4s
-; CHECK-FP16-NEXT: ucvtf v3.4s, v5.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v2.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v3.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v16i8_v16bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v3.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v3.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT: movi v6.4s, #1
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ushll v3.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v4.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: ushr v7.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v18.4s, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v20.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v0.4s, #16
+; CHECK-NOFP16-GI-NEXT: fcmeq v17.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v19.4s, v2.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v23.4s, v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v7.16b, v7.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v18.16b, v18.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v20.16b, v20.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: and v6.16b, v21.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v21.4s, v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v24.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: orr v3.16b, v3.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v4.16b, v4.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-NOFP16-GI-NEXT: add v5.4s, v16.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v7.4s, v19.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: add v16.4s, v23.4s, v20.4s
+; CHECK-NOFP16-GI-NEXT: mvn v17.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: bif v3.16b, v5.16b, v6.16b
+; CHECK-NOFP16-GI-NEXT: mov v5.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bif v2.16b, v7.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: bif v4.16b, v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v5.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i8_v16bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT: ucvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-FP16-GI-NEXT: ucvtf v5.4s, v0.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v3.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v5.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <16 x i8> %a to <16 x bfloat>
ret <16 x bfloat> %c
}
define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) {
-; CHECK-NOFP16-LABEL: stofp_v32i8_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: sshll2 v3.8h, v0.16b, #0
-; CHECK-NOFP16-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: sshll2 v4.8h, v1.16b, #0
-; CHECK-NOFP16-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: sshll v5.4s, v3.4h, #0
-; CHECK-NOFP16-NEXT: sshll v6.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: sshll v7.4s, v4.4h, #0
-; CHECK-NOFP16-NEXT: sshll v16.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v4.4s, v4.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: scvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: scvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: scvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: scvtf v16.4s, v16.4s
-; CHECK-NOFP16-NEXT: scvtf v17.4s, v3.4s
-; CHECK-NOFP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: scvtf v18.4s, v0.4s
-; CHECK-NOFP16-NEXT: scvtf v19.4s, v1.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v3.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: ushr v20.4s, v16.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v20.16b, v20.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v23.16b, v23.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v24.16b, v24.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v25.16b, v25.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v3.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v26.4s, v1.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v20.4s, v20.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v6.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v7.4s, v26.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v16.4s, v20.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v22.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v23.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v7.4s, v24.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v25.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v18.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v17.4s, v6.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v4.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v19.4s, v16.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v4.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v21.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v4.4s, v4.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v16.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v17.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v18.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v19.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v20.4s, v16.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v23.4s, v17.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v24.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v20.16b, v20.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v23.16b, v23.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v24.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v25.16b, v25.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v3.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v26.4s, v1.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v20.4s, v20.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v6.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn v3.4h, v7.4s, v26.4s
+; CHECK-NOFP16-SD-NEXT: addhn v2.4h, v16.4s, v20.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v22.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v23.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v7.4s, v24.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v25.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v18.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v17.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v3.8h, v4.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v2.8h, v19.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: stofp_v32i8_v32bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: sshll v2.8h, v0.8b, #0
-; CHECK-FP16-NEXT: sshll2 v0.8h, v0.16b, #0
-; CHECK-FP16-NEXT: sshll2 v3.8h, v1.16b, #0
-; CHECK-FP16-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-FP16-NEXT: sshll v4.4s, v2.4h, #0
-; CHECK-FP16-NEXT: sshll v5.4s, v0.4h, #0
-; CHECK-FP16-NEXT: sshll v6.4s, v3.4h, #0
-; CHECK-FP16-NEXT: sshll v7.4s, v1.4h, #0
-; CHECK-FP16-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-FP16-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-FP16-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-FP16-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-FP16-NEXT: scvtf v4.4s, v4.4s
-; CHECK-FP16-NEXT: scvtf v5.4s, v5.4s
-; CHECK-FP16-NEXT: scvtf v6.4s, v6.4s
-; CHECK-FP16-NEXT: scvtf v7.4s, v7.4s
-; CHECK-FP16-NEXT: scvtf v16.4s, v0.4s
-; CHECK-FP16-NEXT: scvtf v17.4s, v3.4s
-; CHECK-FP16-NEXT: scvtf v18.4s, v2.4s
-; CHECK-FP16-NEXT: scvtf v19.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v7.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v17.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v19.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: stofp_v32i8_v32bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v3.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: sshll v4.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v5.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v6.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v7.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: scvtf v4.4s, v4.4s
+; CHECK-FP16-SD-NEXT: scvtf v5.4s, v5.4s
+; CHECK-FP16-SD-NEXT: scvtf v6.4s, v6.4s
+; CHECK-FP16-SD-NEXT: scvtf v7.4s, v7.4s
+; CHECK-FP16-SD-NEXT: scvtf v16.4s, v0.4s
+; CHECK-FP16-SD-NEXT: scvtf v17.4s, v3.4s
+; CHECK-FP16-SD-NEXT: scvtf v18.4s, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v19.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v3.4h, v6.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v2.4h, v7.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v16.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v3.8h, v17.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v2.8h, v19.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: sshll v3.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT: sshll v5.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: sshll2 v7.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: sshll v4.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v17.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v22.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v20.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: sshll v4.4s, v5.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: sshll2 v5.4s, v5.8h, #0
+; CHECK-NOFP16-GI-NEXT: sshll v6.4s, v7.4h, #0
+; CHECK-NOFP16-GI-NEXT: sshll2 v7.4s, v7.8h, #0
+; CHECK-NOFP16-GI-NEXT: scvtf v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v19.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v22.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v17.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v23.4s, v20.4s, #16
+; CHECK-NOFP16-GI-NEXT: scvtf v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v22.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: scvtf v5.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v6.16b, v17.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v19.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v21.16b, v21.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v20.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v28.16b, v23.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: scvtf v16.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v22.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v22.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: add v17.4s, v24.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v21.4s, v26.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v27.4s, v28.4s
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v18.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v27.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v23.16b, v20.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v20.4s, v20.4s, v20.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v26.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v18.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v27.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v3.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v28.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v29.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v5.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v16.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v8.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: orr v5.16b, v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v16.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v16.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v22.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bsl v2.16b, v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mov v6.16b, v16.16b
+; CHECK-NOFP16-GI-NEXT: mov v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v9.16b, v26.16b
+; CHECK-NOFP16-GI-NEXT: bif v4.16b, v27.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v17.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bif v5.16b, v28.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v7.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: mov v7.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v16.16b, v31.16b, v25.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v7.16b, v23.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v7.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v7.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v32i8_v32bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT: sshll v4.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v5.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v6.4s, v3.4h, #0
+; CHECK-FP16-GI-NEXT: sshll v7.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-FP16-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: scvtf v4.4s, v4.4s
+; CHECK-FP16-GI-NEXT: scvtf v5.4s, v5.4s
+; CHECK-FP16-GI-NEXT: scvtf v6.4s, v6.4s
+; CHECK-FP16-GI-NEXT: scvtf v7.4s, v7.4s
+; CHECK-FP16-GI-NEXT: scvtf v16.4s, v2.4s
+; CHECK-FP16-GI-NEXT: scvtf v17.4s, v0.4s
+; CHECK-FP16-GI-NEXT: scvtf v18.4s, v3.4s
+; CHECK-FP16-GI-NEXT: scvtf v19.4s, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v2.4h, v6.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v3.4h, v7.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v16.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v17.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v2.8h, v18.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v3.8h, v19.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <32 x i8> %a to <32 x bfloat>
ret <32 x bfloat> %c
}
define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) {
-; CHECK-NOFP16-LABEL: utofp_v32i8_v32bf16:
-; CHECK-NOFP16: // %bb.0: // %entry
-; CHECK-NOFP16-NEXT: ushll2 v3.8h, v0.16b, #0
-; CHECK-NOFP16-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NOFP16-NEXT: ushll2 v4.8h, v1.16b, #0
-; CHECK-NOFP16-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NOFP16-NEXT: movi v2.4s, #1
-; CHECK-NOFP16-NEXT: movi v21.4s, #127, msl #8
-; CHECK-NOFP16-NEXT: ushll v5.4s, v3.4h, #0
-; CHECK-NOFP16-NEXT: ushll v6.4s, v0.4h, #0
-; CHECK-NOFP16-NEXT: ushll v7.4s, v4.4h, #0
-; CHECK-NOFP16-NEXT: ushll v16.4s, v1.4h, #0
-; CHECK-NOFP16-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-NOFP16-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-NOFP16-NEXT: ucvtf v5.4s, v5.4s
-; CHECK-NOFP16-NEXT: ucvtf v6.4s, v6.4s
-; CHECK-NOFP16-NEXT: ucvtf v7.4s, v7.4s
-; CHECK-NOFP16-NEXT: ucvtf v16.4s, v16.4s
-; CHECK-NOFP16-NEXT: ucvtf v17.4s, v3.4s
-; CHECK-NOFP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-NOFP16-NEXT: ucvtf v18.4s, v0.4s
-; CHECK-NOFP16-NEXT: ucvtf v19.4s, v1.4s
-; CHECK-NOFP16-NEXT: ushr v0.4s, v5.4s, #16
-; CHECK-NOFP16-NEXT: ushr v3.4s, v6.4s, #16
-; CHECK-NOFP16-NEXT: ushr v1.4s, v7.4s, #16
-; CHECK-NOFP16-NEXT: ushr v20.4s, v16.4s, #16
-; CHECK-NOFP16-NEXT: ushr v23.4s, v17.4s, #16
-; CHECK-NOFP16-NEXT: ushr v24.4s, v4.4s, #16
-; CHECK-NOFP16-NEXT: ushr v22.4s, v18.4s, #16
-; CHECK-NOFP16-NEXT: ushr v25.4s, v19.4s, #16
-; CHECK-NOFP16-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v3.16b, v3.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v20.16b, v20.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v23.16b, v23.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v24.16b, v24.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v22.16b, v22.16b, v2.16b
-; CHECK-NOFP16-NEXT: and v25.16b, v25.16b, v2.16b
-; CHECK-NOFP16-NEXT: add v0.4s, v0.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v3.4s, v3.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v26.4s, v1.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v20.4s, v20.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn v1.4h, v5.4s, v0.4s
-; CHECK-NOFP16-NEXT: addhn v0.4h, v6.4s, v3.4s
-; CHECK-NOFP16-NEXT: addhn v3.4h, v7.4s, v26.4s
-; CHECK-NOFP16-NEXT: addhn v2.4h, v16.4s, v20.4s
-; CHECK-NOFP16-NEXT: add v5.4s, v22.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v6.4s, v23.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v7.4s, v24.4s, v21.4s
-; CHECK-NOFP16-NEXT: add v16.4s, v25.4s, v21.4s
-; CHECK-NOFP16-NEXT: addhn2 v0.8h, v18.4s, v5.4s
-; CHECK-NOFP16-NEXT: addhn2 v1.8h, v17.4s, v6.4s
-; CHECK-NOFP16-NEXT: addhn2 v3.8h, v4.4s, v7.4s
-; CHECK-NOFP16-NEXT: addhn2 v2.8h, v19.4s, v16.4s
-; CHECK-NOFP16-NEXT: ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32bf16:
+; CHECK-NOFP16-SD: // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-SD-NEXT: movi v21.4s, #127, msl #8
+; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v6.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v4.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v16.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v17.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v18.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v19.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ushr v0.4s, v5.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v3.4s, v6.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v1.4s, v7.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v20.4s, v16.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v23.4s, v17.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v24.4s, v4.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v22.4s, v18.4s, #16
+; CHECK-NOFP16-SD-NEXT: ushr v25.4s, v19.4s, #16
+; CHECK-NOFP16-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v3.16b, v3.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v20.16b, v20.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v23.16b, v23.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v24.16b, v24.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v22.16b, v22.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: and v25.16b, v25.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT: add v0.4s, v0.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v3.4s, v3.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v26.4s, v1.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v20.4s, v20.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn v1.4h, v5.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: addhn v0.4h, v6.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: addhn v3.4h, v7.4s, v26.4s
+; CHECK-NOFP16-SD-NEXT: addhn v2.4h, v16.4s, v20.4s
+; CHECK-NOFP16-SD-NEXT: add v5.4s, v22.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v6.4s, v23.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v7.4s, v24.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: add v16.4s, v25.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v0.8h, v18.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v1.8h, v17.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v3.8h, v4.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: addhn2 v2.8h, v19.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ret
;
-; CHECK-FP16-LABEL: utofp_v32i8_v32bf16:
-; CHECK-FP16: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ushll v2.8h, v0.8b, #0
-; CHECK-FP16-NEXT: ushll2 v0.8h, v0.16b, #0
-; CHECK-FP16-NEXT: ushll2 v3.8h, v1.16b, #0
-; CHECK-FP16-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-FP16-NEXT: ushll v4.4s, v2.4h, #0
-; CHECK-FP16-NEXT: ushll v5.4s, v0.4h, #0
-; CHECK-FP16-NEXT: ushll v6.4s, v3.4h, #0
-; CHECK-FP16-NEXT: ushll v7.4s, v1.4h, #0
-; CHECK-FP16-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-FP16-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-FP16-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-FP16-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-FP16-NEXT: ucvtf v4.4s, v4.4s
-; CHECK-FP16-NEXT: ucvtf v5.4s, v5.4s
-; CHECK-FP16-NEXT: ucvtf v6.4s, v6.4s
-; CHECK-FP16-NEXT: ucvtf v7.4s, v7.4s
-; CHECK-FP16-NEXT: ucvtf v16.4s, v0.4s
-; CHECK-FP16-NEXT: ucvtf v17.4s, v3.4s
-; CHECK-FP16-NEXT: ucvtf v18.4s, v2.4s
-; CHECK-FP16-NEXT: ucvtf v19.4s, v1.4s
-; CHECK-FP16-NEXT: bfcvtn v0.4h, v4.4s
-; CHECK-FP16-NEXT: bfcvtn v1.4h, v5.4s
-; CHECK-FP16-NEXT: bfcvtn v3.4h, v6.4s
-; CHECK-FP16-NEXT: bfcvtn v2.4h, v7.4s
-; CHECK-FP16-NEXT: bfcvtn2 v0.8h, v18.4s
-; CHECK-FP16-NEXT: bfcvtn2 v1.8h, v16.4s
-; CHECK-FP16-NEXT: bfcvtn2 v3.8h, v17.4s
-; CHECK-FP16-NEXT: bfcvtn2 v2.8h, v19.4s
-; CHECK-FP16-NEXT: ret
+; CHECK-FP16-SD-LABEL: utofp_v32i8_v32bf16:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v5.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v6.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v7.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-FP16-SD-NEXT: ucvtf v5.4s, v5.4s
+; CHECK-FP16-SD-NEXT: ucvtf v6.4s, v6.4s
+; CHECK-FP16-SD-NEXT: ucvtf v7.4s, v7.4s
+; CHECK-FP16-SD-NEXT: ucvtf v16.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v17.4s, v3.4s
+; CHECK-FP16-SD-NEXT: ucvtf v18.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v19.4s, v1.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v3.4h, v6.4s
+; CHECK-FP16-SD-NEXT: bfcvtn v2.4h, v7.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v0.8h, v18.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v1.8h, v16.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v3.8h, v17.4s
+; CHECK-FP16-SD-NEXT: bfcvtn2 v2.8h, v19.4s
+; CHECK-FP16-SD-NEXT: ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32bf16:
+; CHECK-NOFP16-GI: // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT: stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b8, -8
+; CHECK-NOFP16-GI-NEXT: .cfi_offset b9, -16
+; CHECK-NOFP16-GI-NEXT: ushll v3.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT: ushll v5.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT: movi v2.4s, #1
+; CHECK-NOFP16-GI-NEXT: ushll2 v7.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-NOFP16-GI-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v6.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v17.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v22.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v20.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: ushll v4.4s, v5.4h, #0
+; CHECK-NOFP16-GI-NEXT: movi v0.4s, #64, lsl #16
+; CHECK-NOFP16-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-NOFP16-GI-NEXT: ushll v6.4s, v7.4h, #0
+; CHECK-NOFP16-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-NOFP16-GI-NEXT: ucvtf v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: ushr v19.4s, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v21.4s, v22.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v17.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v23.4s, v20.4s, #16
+; CHECK-NOFP16-GI-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v22.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: ucvtf v5.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT: orr v6.16b, v17.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v19.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: and v21.16b, v21.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v19.4s, v17.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v20.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v28.16b, v23.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: ucvtf v16.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT: add v29.4s, v3.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: orr v7.16b, v22.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v22.4s, v22.4s, v22.4s
+; CHECK-NOFP16-GI-NEXT: add v17.4s, v24.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: ushr v25.4s, v18.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v21.4s, v26.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT: ushr v26.4s, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v24.4s, v27.4s, v28.4s
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v18.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: ushr v30.4s, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: ushr v27.4s, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v8.4s, v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: and v25.16b, v25.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v23.16b, v20.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v20.4s, v20.4s, v20.4s
+; CHECK-NOFP16-GI-NEXT: and v26.16b, v26.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v31.16b, v18.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v18.4s, v18.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT: and v27.16b, v27.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v9.16b, v3.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: fcmeq v3.4s, v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT: add v25.4s, v28.4s, v25.4s
+; CHECK-NOFP16-GI-NEXT: ushr v28.4s, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: add v26.4s, v29.4s, v26.4s
+; CHECK-NOFP16-GI-NEXT: and v29.16b, v30.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: add v30.4s, v5.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v16.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT: add v27.4s, v8.4s, v27.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v8.4s, v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT: and v2.16b, v28.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT: orr v4.16b, v4.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v18.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: add v28.4s, v30.4s, v29.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v29.4s, v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT: fcmeq v30.4s, v16.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT: orr v5.16b, v5.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: orr v0.16b, v16.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT: mvn v16.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT: mvn v2.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: mvn v19.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mvn v3.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT: mvn v20.16b, v8.16b
+; CHECK-NOFP16-GI-NEXT: mvn v22.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: mvn v29.16b, v30.16b
+; CHECK-NOFP16-GI-NEXT: bsl v2.16b, v6.16b, v17.16b
+; CHECK-NOFP16-GI-NEXT: mov v6.16b, v16.16b
+; CHECK-NOFP16-GI-NEXT: mov v16.16b, v18.16b
+; CHECK-NOFP16-GI-NEXT: bsl v3.16b, v9.16b, v26.16b
+; CHECK-NOFP16-GI-NEXT: bif v4.16b, v27.16b, v20.16b
+; CHECK-NOFP16-GI-NEXT: mov v17.16b, v29.16b
+; CHECK-NOFP16-GI-NEXT: bif v5.16b, v28.16b, v22.16b
+; CHECK-NOFP16-GI-NEXT: bsl v6.16b, v7.16b, v21.16b
+; CHECK-NOFP16-GI-NEXT: mov v7.16b, v19.16b
+; CHECK-NOFP16-GI-NEXT: bsl v16.16b, v31.16b, v25.16b
+; CHECK-NOFP16-GI-NEXT: bsl v17.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v2.4h, v3.4s, #16
+; CHECK-NOFP16-GI-NEXT: bsl v7.16b, v23.16b, v24.16b
+; CHECK-NOFP16-GI-NEXT: shrn v4.4h, v4.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v3.4h, v5.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v6.4h, v6.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v5.4h, v17.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v2.d[1], v4.d[0]
+; CHECK-NOFP16-GI-NEXT: shrn v1.4h, v7.4s, #16
+; CHECK-NOFP16-GI-NEXT: shrn v7.4h, v16.4s, #16
+; CHECK-NOFP16-GI-NEXT: mov v0.d[1], v6.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-NOFP16-GI-NEXT: mov v1.d[1], v7.d[0]
+; CHECK-NOFP16-GI-NEXT: ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT: ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v32i8_v32bf16:
+; CHECK-FP16-GI: // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v5.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v6.4s, v3.4h, #0
+; CHECK-FP16-GI-NEXT: ushll v7.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-FP16-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT: ucvtf v4.4s, v4.4s
+; CHECK-FP16-GI-NEXT: ucvtf v5.4s, v5.4s
+; CHECK-FP16-GI-NEXT: ucvtf v6.4s, v6.4s
+; CHECK-FP16-GI-NEXT: ucvtf v7.4s, v7.4s
+; CHECK-FP16-GI-NEXT: ucvtf v16.4s, v2.4s
+; CHECK-FP16-GI-NEXT: ucvtf v17.4s, v0.4s
+; CHECK-FP16-GI-NEXT: ucvtf v18.4s, v3.4s
+; CHECK-FP16-GI-NEXT: ucvtf v19.4s, v1.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v0.4h, v4.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v1.4h, v5.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v2.4h, v6.4s
+; CHECK-FP16-GI-NEXT: bfcvtn v3.4h, v7.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v0.8h, v16.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v1.8h, v17.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v2.8h, v18.4s
+; CHECK-FP16-GI-NEXT: bfcvtn2 v3.8h, v19.4s
+; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <32 x i8> %a to <32 x bfloat>
ret <32 x bfloat> %c
@@ -3715,7 +5901,3 @@ define <4 x bfloat> @utofp_v4i48_v4f16(<4 x i48> %a) {
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
-; CHECK-FP16-GI: {{.*}}
-; CHECK-FP16-SD: {{.*}}
-; CHECK-NOFP16-GI: {{.*}}
-; CHECK-NOFP16-SD: {{.*}}
More information about the llvm-commits
mailing list