[llvm] [AArch64] Optimize lowering of extending BUILD_VECTOR(EXTRACT_VECTOR_ELT()) (PR #212156)
Usman Nadeem via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 29 15:50:45 PDT 2026
https://github.com/UsmanNadeem updated https://github.com/llvm/llvm-project/pull/212156
>From f65f08394a641c12c2d208af4b8210ebbce88c26 Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 17:41:29 -0700
Subject: [PATCH] [AArch64] Optimize lowering of extending BUILD_VECTOR
Lower them to EXTRACT_SUBVECTOR in more cases, including for fixed vector extracts of predicates.
Change-Id: Ia4fa4c7d8cf5fa7abe1318d00e360a895cdbd109
---
.../Target/AArch64/AArch64ISelLowering.cpp | 179 +-
llvm/test/CodeGen/AArch64/aarch64-load-ext.ll | 9 +-
.../aarch64-neon-vector-insert-uaddlv.ll | 6 +-
.../AArch64/aarch64_be-shuffle-vector.ll | 4 +-
llvm/test/CodeGen/AArch64/add.ll | 4 +-
llvm/test/CodeGen/AArch64/andorxor.ll | 6 +-
llvm/test/CodeGen/AArch64/bitcast-extend.ll | 45 +-
.../CodeGen/AArch64/combine-storetomstore.ll | 401 ++---
llvm/test/CodeGen/AArch64/ctlz.ll | 4 +-
llvm/test/CodeGen/AArch64/extbinopload.ll | 2 +-
llvm/test/CodeGen/AArch64/itofp.ll | 1452 ++++++++---------
llvm/test/CodeGen/AArch64/mul.ll | 6 +-
llvm/test/CodeGen/AArch64/neon-bitcast.ll | 15 +-
llvm/test/CodeGen/AArch64/sub.ll | 4 +-
.../sve-extract-fixed-from-scalable-vector.ll | 11 +-
.../AArch64/sve-extract-fixed-vector.ll | 49 +-
.../AArch64/sve-fixed-length-masked-gather.ll | 18 +-
.../sve-fixed-length-masked-scatter.ll | 25 +-
.../AArch64/sve-fixed-length-reshuffle.ll | 102 +-
.../AArch64/sve-masked-compressstore.ll | 31 +-
...ing-mode-fixed-length-extract-subvector.ll | 24 +-
...streaming-mode-fixed-length-int-extends.ll | 46 +-
...streaming-mode-fixed-length-masked-load.ll | 20 +-
...treaming-mode-fixed-length-masked-store.ll | 26 +-
...e-streaming-mode-fixed-length-reshuffle.ll | 7 +-
llvm/test/CodeGen/AArch64/vec_uaddo.ll | 34 +-
llvm/test/CodeGen/AArch64/vec_umulo.ll | 36 +-
llvm/test/CodeGen/AArch64/vector-fcvt.ll | 290 ++--
llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 157 +-
29 files changed, 1435 insertions(+), 1578 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 70396749e3426..da090b3490961 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23754,37 +23754,162 @@ static SDValue performBuildVectorCombine(SDNode *N,
// (extract_elt_iXX_to_i32 vec Idx+1))
// => (extract_subvector (anyext_iXX_to_i32 vec) Idx)
- // For now, only consider the v2i32 case, which arises as a result of
- // legalization.
- if (VT != MVT::v2i32)
- return SDValue();
-
- SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
- // Reminder, EXTRACT_VECTOR_ELT has the effect of any-extending to its VT.
- if (Elt0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- Elt1->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- // Constant index.
- isa<ConstantSDNode>(Elt0->getOperand(1)) &&
- isa<ConstantSDNode>(Elt1->getOperand(1)) &&
- // Both EXTRACT_VECTOR_ELT from same vector...
- Elt0->getOperand(0) == Elt1->getOperand(0) &&
- // ... and contiguous. First element's index +1 == second element's index.
- Elt0->getConstantOperandVal(1) + 1 == Elt1->getConstantOperandVal(1) &&
- // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
- // ResultType's known minimum vector length.
- Elt0->getConstantOperandVal(1) % VT.getVectorMinNumElements() == 0) {
- SDValue VecToExtend = Elt0->getOperand(0);
- EVT ExtVT = VecToExtend.getValueType().changeVectorElementType(
- *DAG.getContext(), MVT::i32);
- if (!DAG.getTargetLoweringInfo().isTypeLegal(ExtVT))
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ SDValue Op0 = N->getOperand(0);
+
+ auto IsValidBV = [&]() {
+ if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return false;
+ if (!isa<ConstantSDNode>(Op0.getOperand(1)))
+ return false;
+ uint64_t ExtractIdx = Op0.getConstantOperandVal(1);
+ // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
+ // ResultType's known minimum vector length.
+ if (ExtractIdx % VT.getVectorMinNumElements() != 0)
+ return false;
+ // Check that the vector we are extracting from is the same in all extracts
+ // and the indices are sequential.
+ for (const SDValue &Op : drop_begin(N->op_values())) {
+ if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return false;
+ if (Op.getOperand(0) != Op0.getOperand(0))
+ return false;
+ if (!isa<ConstantSDNode>(Op.getOperand(1)))
+ return false;
+ if (Op.getConstantOperandVal(1) != ++ExtractIdx)
+ return false;
+ }
+ return true;
+ };
+
+ if (!IsValidBV())
+ return SDValue();
+
+ SDValue VecToExtract = Op0.getOperand(0);
+ EVT PreExtractVT = VecToExtract.getValueType();
+ uint64_t ExtractIdx = Op0.getConstantOperandVal(1);
+
+ // Scalable predicate to fixed vector extracts.
+ // We want to extend the predicate to the corresponding packed scalable
+ // integer type with the same number of elements. We can then extract a
+ // fixed vector from it, a create an appropriate build vector that can then
+ // be optimized by existing code.
+ //
+ // nxv8i1 -> nxv8i16 -> extract v8i16 -> v4i16 BV.
+ // nxv4i1 -> nxv4i32 -> extract v4i32 -> v4i16 BV.
+ // nxv4i1 -> nxv4i32 -> extract v8i32 if legal -> v8i8 BV.
+ // nxv16i1 -> nxv16i8 -> extract v16i8 -> v4i32 BV.
+ // nxv16i1 -> nxv16i8 -> extract v16i8 -> v4i16 BV.
+ if (PreExtractVT.isScalableVectorOf(MVT::i1) && !DCI.isBeforeLegalize() &&
+ VT.isFixedLengthVector()) {
+ EVT PackedSVIntTy =
+ getPackedSVEVectorVT(PreExtractVT.getVectorElementCount());
+ unsigned ResNumElements = VT.getVectorMinNumElements();
+ unsigned PackedMinElements = PackedSVIntTy.getVectorMinNumElements();
+ // Extracting a bigger vector makes things simpler in terms implementation
+ // using legal types.
+ unsigned ExtractNumElems = std::max(ResNumElements, PackedMinElements);
+ unsigned BVIndex = 0;
+ if (ExtractIdx % ExtractNumElems != 0) {
+ BVIndex = ExtractIdx % PackedMinElements;
+ ExtractIdx = ExtractIdx - BVIndex;
+ }
+ EVT ExtractFixedVT =
+ EVT::getVectorVT(*DAG.getContext(),
+ PackedSVIntTy.getVectorElementType(), ExtractNumElems);
+ if (!TLI.isTypeLegal(ExtractFixedVT) ||
+ (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(ISD::EXTRACT_SUBVECTOR, ExtractFixedVT)))
return SDValue();
- SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend);
- return DAG.getExtractSubvector(DL, MVT::v2i32, Ext,
- Elt0->getConstantOperandVal(1));
+ SDValue FixedExtract = DAG.getExtractSubvector(
+ DL, ExtractFixedVT,
+ DAG.getAnyExtOrTrunc(VecToExtract, DL, PackedSVIntTy), ExtractIdx);
+ EVT ExtractElemTy =
+ ExtractFixedVT.getScalarSizeInBits() > 32 ? MVT::i64 : MVT::i32;
+ SmallVector<SDValue, 16> ExtractElems;
+ for (unsigned Idx = 0; Idx < VT.getVectorNumElements(); ++Idx, ++BVIndex)
+ ExtractElems.push_back(
+ DAG.getExtractVectorElt(DL, ExtractElemTy, FixedExtract, BVIndex));
+ return DAG.getBuildVector(VT, DL, ExtractElems);
}
- return SDValue();
+ if (DCI.isBeforeLegalize() || !VT.isInteger() ||
+ !PreExtractVT.isFixedLengthVector())
+ return SDValue();
+
+ // Not sure if we will ever hit this case.
+ if (PreExtractVT.getScalarSizeInBits() == VT.getScalarSizeInBits())
+ return SDValue();
+
+ auto TryExtractHalf = [&TLI, &DAG, &DCI, &DL](SDValue Vector, unsigned Idx,
+ bool &IsExtractHi) {
+ EVT ExtractVT =
+ Vector.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
+
+ if (!TLI.isTypeLegal(ExtractVT))
+ return SDValue();
+ if (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(ISD::EXTRACT_SUBVECTOR, ExtractVT))
+ return SDValue();
+
+ unsigned ExtractNumElem = ExtractVT.getVectorMinNumElements();
+ IsExtractHi = Idx >= ExtractNumElem;
+ unsigned ExtractIdx = IsExtractHi ? ExtractNumElem : 0;
+ return DAG.getExtractSubvector(DL, ExtractVT, Vector, ExtractIdx);
+ };
+
+ auto TryAnyExtendOrTrunc = [&TLI, &DAG, &DCI, &DL](SDValue Vector,
+ bool IsExtend) {
+ LLVMContext &Context = *DAG.getContext();
+ EVT VT = Vector.getValueType();
+ uint64_t NumBits = VT.getScalarSizeInBits();
+ EVT NewVT = EVT::getVectorVT(
+ Context,
+ EVT::getIntegerVT(Context, IsExtend ? 2 * NumBits : NumBits / 2),
+ VT.getVectorNumElements());
+
+ if (!TLI.isTypeLegal(NewVT))
+ return SDValue();
+ if (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(IsExtend ? ISD::ANY_EXTEND : ISD::TRUNCATE,
+ NewVT))
+ return SDValue();
+ return DAG.getAnyExtOrTrunc(Vector, DL, NewVT);
+ };
+
+ SDValue CurrentVec = VecToExtract;
+ EVT CurrentTy = PreExtractVT;
+
+ // We can't extend a 128bit vector, extract first.
+ if (CurrentTy.is128BitVector() &&
+ VT.getScalarSizeInBits() > CurrentTy.getScalarSizeInBits()) {
+ bool IsExtractHi = false;
+ if (!(CurrentVec = TryExtractHalf(CurrentVec, ExtractIdx, IsExtractHi)))
+ return SDValue();
+ CurrentTy = CurrentVec.getValueType();
+ if (IsExtractHi)
+ ExtractIdx -= CurrentTy.getVectorMinNumElements();
+ }
+
+ // Extend/trunc and extract until we reach the desired element type.
+ while (CurrentTy.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
+ bool IsExtend = VT.getScalarSizeInBits() > CurrentTy.getScalarSizeInBits();
+ if (!(CurrentVec = TryAnyExtendOrTrunc(CurrentVec, IsExtend)))
+ return SDValue();
+
+ CurrentTy = CurrentVec.getValueType();
+ if (CurrentTy.getVectorMinNumElements() == VT.getVectorMinNumElements())
+ continue;
+
+ bool IsExtractHi = false;
+ if (!(CurrentVec = TryExtractHalf(CurrentVec, ExtractIdx, IsExtractHi)))
+ return SDValue();
+ CurrentTy = CurrentVec.getValueType();
+ if (IsExtractHi)
+ ExtractIdx -= CurrentTy.getVectorMinNumElements();
+ }
+ return CurrentVec;
}
// A special combine for the sqdmulh family of instructions.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll b/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
index 0ef2b31d00daa..b910eb6423f35 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
@@ -271,18 +271,15 @@ define <3 x i16> @fsext_v3i16(ptr %a) {
; CHECK-LE-LABEL: fsext_v3i16:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: ldr s0, [x0]
-; CHECK-LE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: fsext_v3i16:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: ldr s0, [x0]
; CHECK-BE-NEXT: rev32 v0.8b, v0.8b
-; CHECK-BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
; CHECK-BE-NEXT: ret
%x = load <3 x i8>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d552397..bbb9e98a39e31 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -281,7 +281,8 @@ define void @insert_vec_v16i8_uaddlv_from_v8i8(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.8b h1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.h v2[0], v1[0]
+; CHECK-NEXT: mov.b v2[0], v1[0]
+; CHECK-NEXT: zip1.8b v2, v2, v2
; CHECK-NEXT: bic.4h v2, #255, lsl #8
; CHECK-NEXT: ushll.4s v2, v2, #0
; CHECK-NEXT: ucvtf.4s v2, v2
@@ -457,7 +458,8 @@ define void @insert_vec_v16i8_uaddlv_from_v4i32(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4s d0, v0
; CHECK-NEXT: stp q2, q2, [x0, #32]
-; CHECK-NEXT: mov.h v1[0], v0[0]
+; CHECK-NEXT: mov.b v1[0], v0[0]
+; CHECK-NEXT: zip1.8b v1, v1, v1
; CHECK-NEXT: bic.4h v1, #255, lsl #8
; CHECK-NEXT: ushll.4s v1, v1, #0
; CHECK-NEXT: ucvtf.4s v1, v1
diff --git a/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll b/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
index f88d88cf1661c..06e215ab703ec 100644
--- a/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
@@ -10,7 +10,7 @@ define <4 x i16> @test_reconstructshuffle(<16 x i8> %a, <16 x i8> %b) nounwind {
; CHECKLE-NEXT: mov v2.b[2], v0.b[2]
; CHECKLE-NEXT: mov v2.b[4], v0.b[1]
; CHECKLE-NEXT: mov v2.b[6], v0.b[0]
-; CHECKLE-NEXT: zip2 v0.8b, v1.8b, v0.8b
+; CHECKLE-NEXT: zip2 v0.8b, v1.8b, v1.8b
; CHECKLE-NEXT: add v0.4h, v2.4h, v0.4h
; CHECKLE-NEXT: bic v0.4h, #255, lsl #8
; CHECKLE-NEXT: ret
@@ -26,7 +26,7 @@ define <4 x i16> @test_reconstructshuffle(<16 x i8> %a, <16 x i8> %b) nounwind {
; CHECKBE-NEXT: mov v2.b[2], v0.b[2]
; CHECKBE-NEXT: mov v2.b[4], v0.b[1]
; CHECKBE-NEXT: mov v2.b[6], v0.b[0]
-; CHECKBE-NEXT: zip2 v0.8b, v1.8b, v0.8b
+; CHECKBE-NEXT: zip2 v0.8b, v1.8b, v1.8b
; CHECKBE-NEXT: add v0.4h, v2.4h, v0.4h
; CHECKBE-NEXT: bic v0.4h, #255, lsl #8
; CHECKBE-NEXT: rev64 v0.4h, v0.4h
diff --git a/llvm/test/CodeGen/AArch64/add.ll b/llvm/test/CodeGen/AArch64/add.ll
index f71c58f16bf36..833a00d44006c 100644
--- a/llvm/test/CodeGen/AArch64/add.ll
+++ b/llvm/test/CodeGen/AArch64/add.ll
@@ -94,9 +94,7 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: add v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
diff --git a/llvm/test/CodeGen/AArch64/andorxor.ll b/llvm/test/CodeGen/AArch64/andorxor.ll
index 6e1f7710b4cd8..1859ac348b0ab 100644
--- a/llvm/test/CodeGen/AArch64/andorxor.ll
+++ b/llvm/test/CodeGen/AArch64/andorxor.ll
@@ -289,7 +289,7 @@ define void @and_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
@@ -340,7 +340,7 @@ define void @or_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
@@ -391,7 +391,7 @@ define void @xor_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: eor v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
diff --git a/llvm/test/CodeGen/AArch64/bitcast-extend.ll b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
index d41f2b3f07311..7b7fe0652eee1 100644
--- a/llvm/test/CodeGen/AArch64/bitcast-extend.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
@@ -62,12 +62,11 @@ define <4 x i64> @z_i32_v4i64(i32 %x) {
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fmov s1, w0
; CHECK-SD-NEXT: movi d0, #0x0000ff000000ff
-; CHECK-SD-NEXT: mov b2, v1.b[0]
-; CHECK-SD-NEXT: mov b3, v1.b[2]
-; CHECK-SD-NEXT: mov v2.b[4], v1.b[1]
-; CHECK-SD-NEXT: mov v3.b[4], v1.b[3]
-; CHECK-SD-NEXT: and v1.8b, v2.8b, v0.8b
-; CHECK-SD-NEXT: and v2.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: mov d2, v1.d[1]
+; CHECK-SD-NEXT: and v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: and v2.8b, v2.8b, v0.8b
; CHECK-SD-NEXT: ushll v0.2d, v1.2s, #0
; CHECK-SD-NEXT: ushll v1.2d, v2.2s, #0
; CHECK-SD-NEXT: ret
@@ -97,20 +96,12 @@ define <4 x i64> @z_i32_v4i64(i32 %x) {
}
define <4 x i16> @s_i32_v4i16(i32 %x) {
-; CHECK-SD-LABEL: s_i32_v4i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: s_i32_v4i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: s_i32_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
%b = bitcast i32 %x to <4 x i8>
%e = sext <4 x i8> %b to <4 x i16>
ret <4 x i16> %e
@@ -157,16 +148,14 @@ define <4 x i64> @s_i32_v4i64(i32 %x) {
; CHECK-SD-LABEL: s_i32_v4i64:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov b1, v0.b[0]
-; CHECK-SD-NEXT: mov b2, v0.b[2]
-; CHECK-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-SD-NEXT: ushll v0.2d, v1.2s, #0
-; CHECK-SD-NEXT: ushll v1.2d, v2.2s, #0
-; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56
-; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #56
+; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56
; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #56
+; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #56
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: s_i32_v4i64:
diff --git a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
index 35e1f3ea3020c..e4b1965e4b76c 100644
--- a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
@@ -160,13 +160,12 @@ define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -184,37 +183,31 @@ define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -243,13 +236,12 @@ define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %
define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -267,37 +259,31 @@ define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1>
define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -347,35 +333,32 @@ define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1>
define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpne p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpne p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpne p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpne p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpne p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: st1w { z2.s }, p2, [x0, x9, lsl #2]
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load
@@ -727,13 +710,12 @@ define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -751,35 +733,32 @@ define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpge p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpge p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpge p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpge p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpge p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpge p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: mov x8, #8 // =0x8
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: st1w { z2.s }, p2, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x
@@ -814,18 +793,17 @@ define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask)
define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_load:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldr q4, [x0]
-; SVE-NEXT: ldr q5, [x0, #16]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldr q4, [x0, #16]
+; SVE-NEXT: ldr q5, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: stp q0, q1, [x0]
; SVE-NEXT: ret
%load = load volatile <8 x i32>, ptr %ptr, align 32
@@ -837,19 +815,18 @@ define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_store:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldp q4, q5, [x0]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldp q5, q4, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
-; SVE-NEXT: str q0, [x0]
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: str q1, [x0, #16]
+; SVE-NEXT: str q0, [x0]
; SVE-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 32
%sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
@@ -874,21 +851,20 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; SVE-NEXT: movi v1.2d, #0000000000000000
; SVE-NEXT: stp q0, q0, [x0]
; SVE-NEXT: bl use_vec
-; SVE-NEXT: zip2 v0.8b, v8.8b, v0.8b
+; SVE-NEXT: ushll v0.8h, v8.8b, #0
; SVE-NEXT: ldp q3, q2, [sp, #16] // 32-byte Folded Reload
-; SVE-NEXT: zip1 v1.8b, v8.8b, v0.8b
-; SVE-NEXT: ushll v0.4s, v0.4h, #0
; SVE-NEXT: ldr d8, [sp, #64] // 8-byte Reload
+; SVE-NEXT: ushll2 v1.4s, v0.8h, #0
+; SVE-NEXT: ushll v0.4s, v0.4h, #0
+; SVE-NEXT: shl v1.4s, v1.4s, #31
; SVE-NEXT: shl v0.4s, v0.4s, #31
-; SVE-NEXT: ushll v1.4s, v1.4h, #0
+; SVE-NEXT: cmlt v1.4s, v1.4s, #0
; SVE-NEXT: cmlt v0.4s, v0.4s, #0
-; SVE-NEXT: shl v1.4s, v1.4s, #31
-; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
; SVE-NEXT: ldr q2, [sp, #48] // 16-byte Reload
; SVE-NEXT: ldr q3, [sp] // 16-byte Reload
-; SVE-NEXT: cmlt v1.4s, v1.4s, #0
-; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
-; SVE-NEXT: stp q1, q0, [x19]
+; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: stp q0, q1, [x19]
; SVE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload
; SVE-NEXT: add sp, sp, #96
; SVE-NEXT: ret
@@ -905,16 +881,14 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {
; SVE-LABEL: test_masked_store_multiple_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v6.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v7.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll v5.8h, v5.8b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v5.4s, v5.4h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
@@ -924,10 +898,10 @@ define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %p
; SVE-NEXT: cmlt v6.4s, v6.4s, #0
; SVE-NEXT: cmpne p1.s, p0/z, z7.s, #0
; SVE-NEXT: cmlt v5.4s, v5.4s, #0
-; SVE-NEXT: ldp q7, q16, [x1]
+; SVE-NEXT: ldp q16, q7, [x1]
; SVE-NEXT: cmpne p2.s, p0/z, z4.s, #0
-; SVE-NEXT: bif v2.16b, v7.16b, v6.16b
-; SVE-NEXT: bif v3.16b, v16.16b, v5.16b
+; SVE-NEXT: bif v3.16b, v7.16b, v6.16b
+; SVE-NEXT: bif v2.16b, v16.16b, v5.16b
; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: st1w { z0.s }, p2, [x0]
; SVE-NEXT: stp q2, q3, [x1]
@@ -946,68 +920,58 @@ define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %p
; SVE: // %bb.0:
; SVE-NEXT: ldp d16, d17, [sp]
; SVE-NEXT: ptrue p1.d, vl2
-; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov x9, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x8, #6 // =0x6
+; SVE-NEXT: mov x8, #2 // =0x2
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov b18, v16.b[0]
-; SVE-NEXT: mov b19, v16.b[4]
-; SVE-NEXT: mov b20, v17.b[4]
-; SVE-NEXT: mov b21, v16.b[6]
-; SVE-NEXT: mov b22, v16.b[2]
-; SVE-NEXT: mov b23, v17.b[6]
-; SVE-NEXT: mov b24, v17.b[2]
-; SVE-NEXT: mov b25, v17.b[0]
-; SVE-NEXT: mov v18.b[4], v16.b[1]
-; SVE-NEXT: mov v19.b[4], v16.b[5]
-; SVE-NEXT: mov v20.b[4], v17.b[5]
-; SVE-NEXT: mov v21.b[4], v16.b[7]
-; SVE-NEXT: mov v22.b[4], v16.b[3]
-; SVE-NEXT: mov v23.b[4], v17.b[7]
-; SVE-NEXT: mov v24.b[4], v17.b[3]
-; SVE-NEXT: mov v25.b[4], v17.b[1]
+; SVE-NEXT: ushll v16.8h, v16.8b, #0
+; SVE-NEXT: ushll v17.8h, v17.8b, #0
+; SVE-NEXT: ushll v18.4s, v16.4h, #0
+; SVE-NEXT: ushll2 v16.4s, v16.8h, #0
+; SVE-NEXT: ushll2 v19.4s, v17.8h, #0
+; SVE-NEXT: ushll v17.4s, v17.4h, #0
+; SVE-NEXT: ushll2 v21.2d, v16.4s, #0
+; SVE-NEXT: ushll2 v20.2d, v18.4s, #0
+; SVE-NEXT: ushll v16.2d, v16.2s, #0
+; SVE-NEXT: ushll2 v22.2d, v19.4s, #0
+; SVE-NEXT: ushll v19.2d, v19.2s, #0
+; SVE-NEXT: ushll2 v23.2d, v17.4s, #0
+; SVE-NEXT: ushll v17.2d, v17.2s, #0
; SVE-NEXT: ushll v18.2d, v18.2s, #0
-; SVE-NEXT: ushll v17.2d, v19.2s, #0
-; SVE-NEXT: ushll v19.2d, v21.2s, #0
-; SVE-NEXT: ushll v21.2d, v25.2s, #0
-; SVE-NEXT: shl v16.2d, v18.2d, #63
-; SVE-NEXT: ushll v18.2d, v20.2s, #0
-; SVE-NEXT: ushll v20.2d, v23.2s, #0
-; SVE-NEXT: shl v17.2d, v17.2d, #63
-; SVE-NEXT: shl v19.2d, v19.2d, #63
; SVE-NEXT: shl v21.2d, v21.2d, #63
-; SVE-NEXT: cmpne p0.d, p1/z, z16.d, #0
-; SVE-NEXT: ushll v16.2d, v22.2s, #0
-; SVE-NEXT: shl v18.2d, v18.2d, #63
-; SVE-NEXT: ushll v22.2d, v24.2s, #0
; SVE-NEXT: shl v20.2d, v20.2d, #63
-; SVE-NEXT: cmpne p2.d, p1/z, z17.d, #0
-; SVE-NEXT: cmpne p3.d, p1/z, z19.d, #0
; SVE-NEXT: shl v16.2d, v16.2d, #63
-; SVE-NEXT: cmlt v17.2d, v18.2d, #0
-; SVE-NEXT: shl v18.2d, v22.2d, #63
-; SVE-NEXT: ldp q19, q22, [x1, #32]
+; SVE-NEXT: shl v19.2d, v19.2d, #63
+; SVE-NEXT: shl v23.2d, v23.2d, #63
+; SVE-NEXT: shl v22.2d, v22.2d, #63
+; SVE-NEXT: shl v17.2d, v17.2d, #63
+; SVE-NEXT: shl v18.2d, v18.2d, #63
+; SVE-NEXT: cmpne p2.d, p1/z, z21.d, #0
+; SVE-NEXT: cmpne p0.d, p1/z, z20.d, #0
; SVE-NEXT: cmpne p4.d, p1/z, z16.d, #0
-; SVE-NEXT: cmlt v16.2d, v20.2d, #0
-; SVE-NEXT: cmlt v20.2d, v21.2d, #0
-; SVE-NEXT: cmlt v18.2d, v18.2d, #0
-; SVE-NEXT: bif v6.16b, v19.16b, v17.16b
-; SVE-NEXT: ldp q17, q19, [x1]
-; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
-; SVE-NEXT: mov x9, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p3, [x0, x8, lsl #3]
-; SVE-NEXT: mov v2.16b, v16.16b
-; SVE-NEXT: mov v3.16b, v20.16b
-; SVE-NEXT: st1d { z1.d }, p4, [x0, x9, lsl #3]
-; SVE-NEXT: mov v1.16b, v18.16b
-; SVE-NEXT: st1d { z0.d }, p0, [x0]
-; SVE-NEXT: bsl v2.16b, v7.16b, v22.16b
-; SVE-NEXT: bsl v3.16b, v4.16b, v17.16b
-; SVE-NEXT: bsl v1.16b, v5.16b, v19.16b
-; SVE-NEXT: stp q6, q2, [x1, #32]
-; SVE-NEXT: stp q3, q1, [x1]
+; SVE-NEXT: cmlt v16.2d, v19.2d, #0
+; SVE-NEXT: cmlt v23.2d, v23.2d, #0
+; SVE-NEXT: cmlt v22.2d, v22.2d, #0
+; SVE-NEXT: cmlt v17.2d, v17.2d, #0
+; SVE-NEXT: ldp q21, q20, [x1, #32]
+; SVE-NEXT: cmpne p3.d, p1/z, z18.d, #0
+; SVE-NEXT: ldp q19, q18, [x1]
+; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov v3.16b, v16.16b
+; SVE-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]
+; SVE-NEXT: mov v1.16b, v23.16b
+; SVE-NEXT: bif v7.16b, v20.16b, v22.16b
+; SVE-NEXT: st1d { z2.d }, p4, [x0, x9, lsl #3]
+; SVE-NEXT: mov v2.16b, v17.16b
+; SVE-NEXT: bsl v3.16b, v6.16b, v21.16b
+; SVE-NEXT: st1d { z0.d }, p3, [x0]
+; SVE-NEXT: bsl v1.16b, v5.16b, v18.16b
+; SVE-NEXT: bsl v2.16b, v4.16b, v19.16b
+; SVE-NEXT: stp q3, q7, [x1, #32]
+; SVE-NEXT: stp q2, q1, [x1]
; SVE-NEXT: ret
%load = load <8 x i64>, ptr %ptr1, align 32
%load2 = load <8 x i64>, ptr %ptr2, align 32
@@ -1066,21 +1030,20 @@ define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i
define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: add x8, x0, #1
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: add x9, x0, #17
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: add x9, x0, #1
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #0
; SVE-NEXT: cmpne p2.s, p0/z, z2.s, #0
-; SVE-NEXT: st1w { z0.s }, p1, [x8]
-; SVE-NEXT: st1w { z1.s }, p2, [x9]
+; SVE-NEXT: st1w { z1.s }, p1, [x8]
+; SVE-NEXT: st1w { z0.s }, p2, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i32 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
@@ -1093,40 +1056,34 @@ define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i
define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: add x8, x0, #33
-; SVE-NEXT: mov b7, v4.b[0]
-; SVE-NEXT: mov b16, v4.b[2]
-; SVE-NEXT: add x9, x0, #49
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: add x8, x0, #49
+; SVE-NEXT: add x9, x0, #33
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[1]
-; SVE-NEXT: mov v16.b[4], v4.b[3]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x8]
-; SVE-NEXT: add x8, x0, #1
-; SVE-NEXT: st1d { z3.d }, p2, [x9]
-; SVE-NEXT: add x9, x0, #17
-; SVE-NEXT: st1d { z0.d }, p3, [x8]
-; SVE-NEXT: st1d { z1.d }, p4, [x9]
+; SVE-NEXT: st1d { z3.d }, p1, [x8]
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: st1d { z2.d }, p2, [x9]
+; SVE-NEXT: add x9, x0, #1
+; SVE-NEXT: st1d { z1.d }, p3, [x8]
+; SVE-NEXT: st1d { z0.d }, p4, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i64 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
index 968fd6d7c9813..6f89dfafd9664 100644
--- a/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -38,9 +38,9 @@ define void @v3i8(ptr %p1) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: movi v0.4h, #8
; CHECK-SD-NEXT: ldr s1, [x0]
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: movi v0.4h, #8
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT: clz v1.4h, v1.4h
; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll
index 2641bca482bfd..790cfb4736567 100644
--- a/llvm/test/CodeGen/AArch64/extbinopload.ll
+++ b/llvm/test/CodeGen/AArch64/extbinopload.ll
@@ -1367,8 +1367,8 @@ define <4 x i32> @atomic(ptr %p) {
; CHECK-NEXT: ldar w8, [x0]
; CHECK-NEXT: ldr s0, [x0, #4]
; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: zip1 v1.8b, v1.8b, v0.8b
; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-NEXT: ushll v0.4s, v0.4h, #3
; CHECK-NEXT: bic v1.4h, #255, lsl #8
; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index c61b38d73c29d..147377a6434e2 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -2355,60 +2355,52 @@ entry:
define <8 x double> @stofp_v8i8_v8f64(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-SD-NEXT: mov b1, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-SD-NEXT: mov b1, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
-; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v5.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f64:
@@ -2448,54 +2440,46 @@ entry:
define <8 x double> @utofp_v8i8_v8f64(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[6]
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NOFP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d3, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d4, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[6]
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-FP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d3, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d4, v0.d[1]
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v1.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f64:
@@ -2535,108 +2519,94 @@ entry:
define <16 x double> @stofp_v16i8_v16f64(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b6, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b7, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b16, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d5, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v7.2s, v7.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v20.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b6, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b7, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b16, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-FP16-SD-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d5, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v0.d[1]
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: shl v7.2s, v7.2s, #24
; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-FP16-SD-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v16.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v17.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v5.2d
+; CHECK-FP16-SD-NEXT: scvtf v7.2d, v19.2d
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v20.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f64:
@@ -2698,94 +2668,80 @@ entry:
define <16 x double> @utofp_v16i8_v16f64(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b6, v0.b[6]
+; CHECK-NOFP16-SD-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-NOFP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov b7, v2.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b16, v2.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b17, v2.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b18, v2.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d5, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v19.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b6, v0.b[6]
+; CHECK-FP16-SD-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-FP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov b7, v2.b[0]
-; CHECK-FP16-SD-NEXT: mov b16, v2.b[2]
-; CHECK-FP16-SD-NEXT: mov b17, v2.b[4]
-; CHECK-FP16-SD-NEXT: mov b18, v2.b[6]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-FP16-SD-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d5, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v0.d[1]
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v19.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f64:
@@ -2847,222 +2803,194 @@ entry:
define <32 x double> @stofp_v32i8_v32f64(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b5, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b20, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b21, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b18, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b22, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov d16, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b2, v3.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b4, v3.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b6, v3.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b7, v3.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v20.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v22.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov b23, v16.b[0]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v3.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v3.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v3.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v3.b[7]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d6, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d18, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: shl v5.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v17.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v20.2s, v20.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b24, v16.b[4]
-; CHECK-NOFP16-SD-NEXT: mov v23.b[4], v16.b[1]
-; CHECK-NOFP16-SD-NEXT: shl v18.2s, v18.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d23, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d21, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d22, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v19.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d7, v7.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v16.2s, v16.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v17.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v6.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b7, v16.b[2]
+; CHECK-NOFP16-SD-NEXT: shl v18.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v23.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v20.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d17, v17.d[1]
+; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v16.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v20.2s, v20.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b21, v16.b[6]
-; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v18.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v21.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov v24.b[4], v16.b[5]
-; CHECK-NOFP16-SD-NEXT: sshr v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v16.b[3]
-; CHECK-NOFP16-SD-NEXT: sshll v20.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v16.b[7]
+; CHECK-NOFP16-SD-NEXT: sshll v18.2d, v18.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshr v23.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v22.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v19.2d, v19.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v20.2d, v20.2d
-; CHECK-NOFP16-SD-NEXT: shl v22.2s, v24.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v18.2d, v18.2d
; CHECK-NOFP16-SD-NEXT: shl v7.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: stp q17, q5, [x8, #160]
-; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v18.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v17.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v17.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: shl v21.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: scvtf v19.2d, v19.2d
-; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
-; CHECK-NOFP16-SD-NEXT: stp q0, q20, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v20.2s, #24
+; CHECK-NOFP16-SD-NEXT: stp q2, q18, [x8, #192]
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q4, q6, [x8, #224]
+; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v23.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v17.2s, v19.2s, #24
; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: sshr v18.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v20.2s, v22.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q16, q19, [x8, #32]
-; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v18.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: stp q5, q3, [x8]
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: stp q1, q6, [x8, #160]
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v17.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: stp q4, q5, [x8, #96]
-; CHECK-NOFP16-SD-NEXT: stp q3, q0, [x8, #192]
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v1.2d
-; CHECK-NOFP16-SD-NEXT: stp q16, q17, [x8, #224]
-; CHECK-NOFP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: stp q3, q4, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: stp q3, q2, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q0, q1, [x8]
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b5, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov b20, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b21, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b18, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b22, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov d16, v1.d[1]
-; CHECK-FP16-SD-NEXT: mov b2, v3.b[0]
-; CHECK-FP16-SD-NEXT: mov b4, v3.b[2]
-; CHECK-FP16-SD-NEXT: mov b6, v3.b[4]
-; CHECK-FP16-SD-NEXT: mov b7, v3.b[6]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov v20.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v22.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov b23, v16.b[0]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v3.b[1]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v3.b[3]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v3.b[5]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v3.b[7]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v17.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v7.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: mov d6, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-FP16-SD-NEXT: mov d18, v2.d[1]
+; CHECK-FP16-SD-NEXT: shl v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: shl v5.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v17.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: shl v20.2s, v20.2s, #24
-; CHECK-FP16-SD-NEXT: mov b24, v16.b[4]
-; CHECK-FP16-SD-NEXT: mov v23.b[4], v16.b[1]
-; CHECK-FP16-SD-NEXT: shl v18.2s, v18.2s, #24
-; CHECK-FP16-SD-NEXT: shl v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT: mov d23, v1.d[1]
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: mov d21, v0.d[1]
+; CHECK-FP16-SD-NEXT: mov d22, v3.d[1]
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v19.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT: mov d7, v7.d[1]
+; CHECK-FP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: shl v16.2s, v16.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: sshr v5.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v17.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: shl v6.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: mov b7, v16.b[2]
+; CHECK-FP16-SD-NEXT: shl v18.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v23.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: shl v20.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: mov d17, v17.d[1]
+; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v16.2s, v16.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v20.2s, v20.2s, #24
-; CHECK-FP16-SD-NEXT: mov b21, v16.b[6]
-; CHECK-FP16-SD-NEXT: sshll v17.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-FP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v18.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT: shl v21.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: mov v24.b[4], v16.b[5]
-; CHECK-FP16-SD-NEXT: sshr v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v16.b[3]
-; CHECK-FP16-SD-NEXT: sshll v20.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v16.b[7]
+; CHECK-FP16-SD-NEXT: sshll v18.2d, v18.2s, #0
+; CHECK-FP16-SD-NEXT: sshr v23.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v16.2s, v22.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v19.2d, v19.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v20.2d, v20.2d
-; CHECK-FP16-SD-NEXT: shl v22.2s, v24.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v18.2d, v18.2d
; CHECK-FP16-SD-NEXT: shl v7.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: stp q17, q5, [x8, #160]
-; CHECK-FP16-SD-NEXT: sshr v5.2s, v18.2s, #24
-; CHECK-FP16-SD-NEXT: shl v17.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: shl v17.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: shl v21.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: scvtf v19.2d, v19.2d
-; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
-; CHECK-FP16-SD-NEXT: stp q0, q20, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-FP16-SD-NEXT: sshr v5.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v16.2s, v20.2s, #24
+; CHECK-FP16-SD-NEXT: stp q2, q18, [x8, #192]
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: stp q4, q6, [x8, #224]
+; CHECK-FP16-SD-NEXT: sshr v4.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v23.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v7.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v2.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v7.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v17.2s, v19.2s, #24
; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: sshr v18.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v20.2s, v22.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: stp q16, q19, [x8, #32]
-; CHECK-FP16-SD-NEXT: sshll v17.2d, v18.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v16.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-SD-NEXT: stp q5, q3, [x8]
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v7.2d
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v6.2d
-; CHECK-FP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT: stp q1, q6, [x8, #160]
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v17.2s, #0
+; CHECK-FP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: stp q4, q5, [x8, #96]
-; CHECK-FP16-SD-NEXT: stp q3, q0, [x8, #192]
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
-; CHECK-FP16-SD-NEXT: stp q16, q17, [x8, #224]
-; CHECK-FP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT: stp q3, q4, [x8, #64]
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v16.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v6.2d
+; CHECK-FP16-SD-NEXT: stp q3, q2, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q0, q1, [x8]
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f64:
@@ -3184,192 +3112,164 @@ entry:
define <32 x double> @utofp_v32i8_v32f64(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov b6, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b7, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b16, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b20, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: movi d5, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov b24, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b25, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov b18, v3.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b21, v3.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b23, v3.b[4]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov b1, v3.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v20.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v24.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v25.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v6.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v3.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v3.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v23.b[4], v3.b[5]
-; CHECK-NOFP16-SD-NEXT: and v16.8b, v16.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v17.8b, v17.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v1.b[4], v3.b[7]
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v19.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: movi d6, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v19.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d17, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v5.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d20, v19.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d21, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d23, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: mov d22, v18.d[1]
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v16.8b, v16.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v17.8b, v17.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: and v20.8b, v20.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v20.8b, v20.8b, v5.8b
; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b4, v2.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b22, v2.b[2]
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b19, v2.b[4]
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: ushll v20.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b7, v2.b[6]
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-NOFP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: and v24.8b, v24.8b, v5.8b
; CHECK-NOFP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v2.b[1]
-; CHECK-NOFP16-SD-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-NOFP16-SD-NEXT: mov v22.b[4], v2.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v2.b[5]
-; CHECK-NOFP16-SD-NEXT: stp q3, q6, [x8, #160]
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v25.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v1.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v2.b[7]
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v23.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v24.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q17, q16, [x8, #128]
-; CHECK-NOFP16-SD-NEXT: and v16.8b, v21.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: stp q20, q0, [x8, #32]
-; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v18.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v20.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v23.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: stp q2, q17, [x8, #192]
+; CHECK-NOFP16-SD-NEXT: stp q4, q7, [x8, #224]
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v19.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d4, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: and v19.8b, v21.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v1.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v19.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v4.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: and v16.8b, v22.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v6.8b, v18.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v17.8b, v19.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v18.8b, v22.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: stp q7, q4, [x8, #160]
+; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v16.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: stp q6, q3, [x8]
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v18.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q2, q1, [x8, #224]
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: stp q0, q16, [x8, #192]
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: stp q1, q5, [x8, #96]
-; CHECK-NOFP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: stp q3, q17, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: stp q1, q2, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q0, q3, [x8]
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov b6, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov b7, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: mov b16, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b20, v0.b[4]
-; CHECK-FP16-SD-NEXT: movi d5, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov b24, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b25, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov b18, v3.b[0]
-; CHECK-FP16-SD-NEXT: mov b21, v3.b[2]
-; CHECK-FP16-SD-NEXT: mov b23, v3.b[4]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov b1, v3.b[6]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v20.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v24.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v25.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: and v6.8b, v6.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v3.b[1]
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v3.b[3]
-; CHECK-FP16-SD-NEXT: mov v23.b[4], v3.b[5]
-; CHECK-FP16-SD-NEXT: and v16.8b, v16.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v17.8b, v17.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v1.b[4], v3.b[7]
-; CHECK-FP16-SD-NEXT: and v3.8b, v19.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: movi d6, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v19.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v18.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d17, v2.d[1]
+; CHECK-FP16-SD-NEXT: and v5.8b, v5.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d20, v19.d[1]
+; CHECK-FP16-SD-NEXT: mov d21, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d23, v0.d[1]
+; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: mov d22, v18.d[1]
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v16.8b, v16.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: and v17.8b, v17.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: and v20.8b, v20.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: and v20.8b, v20.8b, v5.8b
; CHECK-FP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: mov b4, v2.b[0]
-; CHECK-FP16-SD-NEXT: mov b22, v2.b[2]
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
; CHECK-FP16-SD-NEXT: ushll v17.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: mov b19, v2.b[4]
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v7.2d
-; CHECK-FP16-SD-NEXT: ushll v20.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: mov b7, v2.b[6]
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-FP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: and v24.8b, v24.8b, v5.8b
; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v2.b[1]
-; CHECK-FP16-SD-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-FP16-SD-NEXT: mov v22.b[4], v2.b[3]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v2.b[5]
-; CHECK-FP16-SD-NEXT: stp q3, q6, [x8, #160]
-; CHECK-FP16-SD-NEXT: and v6.8b, v25.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v1.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v2.b[7]
-; CHECK-FP16-SD-NEXT: and v2.8b, v23.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v24.2s, #0
-; CHECK-FP16-SD-NEXT: stp q17, q16, [x8, #128]
-; CHECK-FP16-SD-NEXT: and v16.8b, v21.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-FP16-SD-NEXT: stp q20, q0, [x8, #32]
-; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: and v0.8b, v18.8b, v5.8b
+; CHECK-FP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-FP16-SD-NEXT: ushll v16.2d, v20.2s, #0
+; CHECK-FP16-SD-NEXT: and v5.8b, v23.8b, v6.8b
+; CHECK-FP16-SD-NEXT: stp q2, q17, [x8, #192]
+; CHECK-FP16-SD-NEXT: stp q4, q7, [x8, #224]
+; CHECK-FP16-SD-NEXT: and v7.8b, v19.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d4, v1.d[1]
+; CHECK-FP16-SD-NEXT: and v19.8b, v21.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: and v1.8b, v1.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v19.2s, #0
+; CHECK-FP16-SD-NEXT: and v2.8b, v4.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v16.2d
+; CHECK-FP16-SD-NEXT: and v16.8b, v22.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v6.8b, v18.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-FP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v17.8b, v19.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: and v18.8b, v22.8b, v5.8b
+; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: stp q7, q4, [x8, #160]
+; CHECK-FP16-SD-NEXT: ushll v4.2d, v16.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: ushll v5.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v7.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: stp q6, q3, [x8]
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v18.2s, #0
-; CHECK-FP16-SD-NEXT: stp q2, q1, [x8, #224]
-; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v7.2d
-; CHECK-FP16-SD-NEXT: stp q0, q16, [x8, #192]
-; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v4.2d
-; CHECK-FP16-SD-NEXT: stp q1, q5, [x8, #96]
-; CHECK-FP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT: stp q3, q17, [x8, #64]
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: stp q1, q2, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q0, q3, [x8]
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f64:
@@ -4763,30 +4663,20 @@ entry:
define <8 x float> @stofp_v8i8_v8f32(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v1.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.4s, v1.4s
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v0.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f32:
@@ -4814,26 +4704,28 @@ entry:
define <8 x float> @utofp_v8i8_v8f32(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v1.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v1.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f32:
@@ -4861,52 +4753,30 @@ entry:
define <16 x float> @stofp_v16i8_v16f32(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v2.4s
-; CHECK-NOFP16-SD-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v3.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v4.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-FP16-SD-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-FP16-SD-NEXT: scvtf v0.4s, v2.4s
-; CHECK-FP16-SD-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v4.4s
-; CHECK-FP16-SD-NEXT: scvtf v2.4s, v3.4s
-; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v4.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f32:
@@ -4944,44 +4814,46 @@ entry:
define <16 x float> @utofp_v16i8_v16f32(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v5.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v5.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f32:
@@ -5019,94 +4891,50 @@ entry:
define <32 x float> @stofp_v32i8_v32f32(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v4.4h, v4.4h, #8
-; CHECK-NOFP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v4.4h, v4.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v5.4h, v5.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v6.4h, v6.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v7.4h, v7.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v4.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshr v5.4h, v5.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v6.4h, v6.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v7.4h, v7.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v16.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v5.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v6.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v17.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v16.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v17.4s, v3.8h, #0
; CHECK-NOFP16-SD-NEXT: sshll v18.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v19.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v5.4s
; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v6.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v17.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v5.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v18.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v7.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v18.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v4.4h, v4.4h, #8
-; CHECK-FP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v4.4h, v4.4h, #8
-; CHECK-FP16-SD-NEXT: shl v5.4h, v5.4h, #8
-; CHECK-FP16-SD-NEXT: shl v6.4h, v6.4h, #8
-; CHECK-FP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: shl v7.4h, v7.4h, #8
-; CHECK-FP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v4.4s, v4.4h, #0
-; CHECK-FP16-SD-NEXT: sshr v5.4h, v5.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v6.4h, v6.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v7.4h, v7.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v16.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.4s, v4.4s
-; CHECK-FP16-SD-NEXT: sshll v5.4s, v5.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v6.4s, v6.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v17.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v7.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v16.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v17.4s, v3.8h, #0
; CHECK-FP16-SD-NEXT: sshll v18.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v19.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v16.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v4.4s
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
; CHECK-FP16-SD-NEXT: scvtf v2.4s, v6.4s
-; CHECK-FP16-SD-NEXT: scvtf v3.4s, v17.4s
-; CHECK-FP16-SD-NEXT: scvtf v4.4s, v5.4s
-; CHECK-FP16-SD-NEXT: scvtf v5.4s, v18.4s
-; CHECK-FP16-SD-NEXT: scvtf v6.4s, v7.4s
-; CHECK-FP16-SD-NEXT: scvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT: scvtf v4.4s, v7.4s
+; CHECK-FP16-SD-NEXT: scvtf v7.4s, v16.4s
+; CHECK-FP16-SD-NEXT: scvtf v5.4s, v17.4s
+; CHECK-FP16-SD-NEXT: scvtf v6.4s, v18.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f32:
@@ -5164,78 +4992,82 @@ entry:
define <32 x float> @utofp_v32i8_v32f32(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v4.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d5, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v4.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v5.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v6.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v7.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v16.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.4s, v4.4h, #0
; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v19.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v16.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v5.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v6.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v17.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v18.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v7.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v6.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v19.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v4.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d5, v0.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v1.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v4.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v5.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v6.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v7.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v16.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v4.4s
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v17.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v16.4s, v4.4h, #0
; CHECK-FP16-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v17.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v18.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v19.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v16.4s
-; CHECK-FP16-SD-NEXT: ucvtf v4.4s, v5.4s
-; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v6.4s
-; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v17.4s
-; CHECK-FP16-SD-NEXT: ucvtf v5.4s, v18.4s
-; CHECK-FP16-SD-NEXT: ucvtf v6.4s, v7.4s
-; CHECK-FP16-SD-NEXT: ucvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT: ushll v18.4s, v6.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v6.4s, v17.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v16.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v5.4s
+; CHECK-FP16-SD-NEXT: ucvtf v7.4s, v18.4s
+; CHECK-FP16-SD-NEXT: ucvtf v5.4s, v19.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f32:
diff --git a/llvm/test/CodeGen/AArch64/mul.ll b/llvm/test/CodeGen/AArch64/mul.ll
index dfd6f9801e54b..0ef13136c1152 100644
--- a/llvm/test/CodeGen/AArch64/mul.ll
+++ b/llvm/test/CodeGen/AArch64/mul.ll
@@ -96,10 +96,8 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: mul v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: xtn v1.8b, v0.8h
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
; CHECK-SD-NEXT: stur b0, [x0, #2]
diff --git a/llvm/test/CodeGen/AArch64/neon-bitcast.ll b/llvm/test/CodeGen/AArch64/neon-bitcast.ll
index c6aa8701e1721..6b43dd23f45aa 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitcast.ll
@@ -554,19 +554,18 @@ define <4 x i8> @bitcast_i32_to_v4i8(i32 %word) {
define <2 x i8> @bitcast_i16_to_v2i8(i16 %word) {
; CHECK-LE-LABEL: bitcast_i16_to_v2i8:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: fmov s1, w0
-; CHECK-LE-NEXT: mov b0, v1.b[0]
-; CHECK-LE-NEXT: mov v0.b[4], v1.b[1]
-; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-LE-NEXT: fmov s0, w0
+; CHECK-LE-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-LE-NEXT: zip1 v0.4h, v0.4h, v0.4h
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: bitcast_i16_to_v2i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: fmov s0, w0
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: mov b1, v0.b[0]
-; CHECK-BE-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-BE-NEXT: rev64 v0.2s, v1.2s
+; CHECK-BE-NEXT: rev16 v0.8b, v0.8b
+; CHECK-BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-BE-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
; CHECK-BE-NEXT: ret
%ret = bitcast i16 %word to <2 x i8>
ret <2 x i8> %ret
diff --git a/llvm/test/CodeGen/AArch64/sub.ll b/llvm/test/CodeGen/AArch64/sub.ll
index 4516cf821af20..34d7b57c8bf45 100644
--- a/llvm/test/CodeGen/AArch64/sub.ll
+++ b/llvm/test/CodeGen/AArch64/sub.ll
@@ -94,9 +94,7 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: sub v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: usubl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index 2a082b34193d8..2b01551222771 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -147,15 +147,8 @@ define <2 x float> @extract_v2f32_nxv16f32_2(<vscale x 16 x float> %arg) {
define <4 x i1> @extract_v4i1_nxv32i1_0(<vscale x 32 x i1> %arg) {
; CHECK-LABEL: extract_v4i1_nxv32i1_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
-; CHECK-NEXT: umov w8, v1.b[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: umov w9, v1.b[2]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: umov w8, v1.b[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b
; CHECK-NEXT: ret
%ext = call <4 x i1> @llvm.vector.extract.v4i1.nxv32i1(<vscale x 32 x i1> %arg, i64 0)
ret <4 x i1> %ext
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
index 0be662720c416..dc8eef3b056d1 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
@@ -238,8 +238,7 @@ define <2 x i1> @extract_v2i1_nxv2i1(<vscale x 2 x i1> %inmask) {
; CHECK-LABEL: extract_v2i1_nxv2i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1
-; CHECK-NEXT: mov v0.s[1], v0.s[2]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: ret
%mask = call <2 x i1> @llvm.vector.extract.v2i1.nxv2i1(<vscale x 2 x i1> %inmask, i64 0)
ret <2 x i1> %mask
@@ -248,15 +247,8 @@ define <2 x i1> @extract_v2i1_nxv2i1(<vscale x 2 x i1> %inmask) {
define <4 x i1> @extract_v4i1_nxv4i1(<vscale x 4 x i1> %inmask) {
; CHECK-LABEL: extract_v4i1_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: mov w9, v1.s[2]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: mov w8, v1.s[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
%mask = call <4 x i1> @llvm.vector.extract.v4i1.nxv4i1(<vscale x 4 x i1> %inmask, i64 0)
ret <4 x i1> %mask
@@ -265,23 +257,8 @@ define <4 x i1> @extract_v4i1_nxv4i1(<vscale x 4 x i1> %inmask) {
define <8 x i1> @extract_v8i1_nxv8i1(<vscale x 8 x i1> %inmask) {
; CHECK-LABEL: extract_v8i1_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.h, p0/z, #1 // =0x1
-; CHECK-NEXT: umov w8, v1.h[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: umov w9, v1.h[2]
-; CHECK-NEXT: mov v0.b[1], w8
-; CHECK-NEXT: umov w8, v1.h[3]
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: umov w9, v1.h[4]
-; CHECK-NEXT: mov v0.b[3], w8
-; CHECK-NEXT: umov w8, v1.h[5]
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: umov w9, v1.h[6]
-; CHECK-NEXT: mov v0.b[5], w8
-; CHECK-NEXT: umov w8, v1.h[7]
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: mov v0.b[7], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: ret
%mask = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> %inmask, i64 0)
ret <8 x i1> %mask
@@ -293,22 +270,6 @@ define <16 x i1> @extract_v16i1_nxv16i1(<vscale x 16 x i1> %inmask) {
; CHECK-LABEL: extract_v16i1_nxv16i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
-; CHECK-NEXT: mov v1.16b, v0.16b
-; CHECK-NEXT: mov v0.b[1], v0.b[1]
-; CHECK-NEXT: mov v0.b[2], v1.b[2]
-; CHECK-NEXT: mov v0.b[3], v1.b[3]
-; CHECK-NEXT: mov v0.b[4], v1.b[4]
-; CHECK-NEXT: mov v0.b[5], v1.b[5]
-; CHECK-NEXT: mov v0.b[6], v1.b[6]
-; CHECK-NEXT: mov v0.b[7], v1.b[7]
-; CHECK-NEXT: mov v0.b[8], v1.b[8]
-; CHECK-NEXT: mov v0.b[9], v1.b[9]
-; CHECK-NEXT: mov v0.b[10], v1.b[10]
-; CHECK-NEXT: mov v0.b[11], v1.b[11]
-; CHECK-NEXT: mov v0.b[12], v1.b[12]
-; CHECK-NEXT: mov v0.b[13], v1.b[13]
-; CHECK-NEXT: mov v0.b[14], v1.b[14]
-; CHECK-NEXT: mov v0.b[15], v1.b[15]
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%mask = call <16 x i1> @llvm.vector.extract.v16i1.nxv16i1(<vscale x 16 x i1> %inmask, i64 0)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
index eb938224b5375..03a974ad3d1a3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
@@ -58,20 +58,18 @@ define void @masked_gather_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
define void @masked_gather_v8i8(ptr %a, ptr %b) #0 {
; VBITS_GE_256-LABEL: masked_gather_v8i8:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldr d0, [x0]
+; VBITS_GE_256-NEXT: ldp s0, s1, [x0]
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0
-; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #8
-; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: sshr v0.4h, v0.4h, #8
-; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0
+; VBITS_GE_256-NEXT: ushll v1.8h, v1.8b, #0
+; VBITS_GE_256-NEXT: mov v0.d[1], v1.d[0]
+; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: mov d1, v0.d[1]
; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s
+; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z1.d]
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
index 07239f3c52204..96c5857a4ad67 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
@@ -58,25 +58,20 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ldr d0, [x0]
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: cmeq v1.8b, v0.8b, #0
-; VBITS_GE_256-NEXT: zip1 v3.8b, v0.8b, v0.8b
+; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0
; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: zip1 v2.8b, v1.8b, v0.8b
-; VBITS_GE_256-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; VBITS_GE_256-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
-; VBITS_GE_256-NEXT: shl v2.4h, v2.4h, #8
-; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: cmeq v1.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h
+; VBITS_GE_256-NEXT: mov d0, v0.d[1]
+; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h
+; VBITS_GE_256-NEXT: mov d1, v1.d[1]
; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
-; VBITS_GE_256-NEXT: sshr v2.4h, v2.4h, #8
-; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
-; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s
-; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z2.d]
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
index 41e4a38fad90b..f5296aea2e4a6 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
@@ -1,32 +1,112 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
+; RUN: llc -mattr=+sve < %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
; == Matching first N elements ==
-define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) #0 {
+define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) {
; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+ %el0 = extractelement <vscale x 4 x i1> %a, i32 0
+ %el1 = extractelement <vscale x 4 x i1> %a, i32 1
+ %el2 = extractelement <vscale x 4 x i1> %a, i32 2
+ %el3 = extractelement <vscale x 4 x i1> %a, i32 3
+ %v0 = insertelement <4 x i1> poison, i1 %el0, i32 0
+ %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
+ %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
+ %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
+ ret <4 x i1> %v3
+}
+
+define <4 x i1> @reshuffle_v4i1_nxv4i1_beyondMinVL(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: reshuffle_v4i1_nxv4i1_beyondMinVL:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+ %el0 = extractelement <vscale x 4 x i1> %a, i32 4
+ %el1 = extractelement <vscale x 4 x i1> %a, i32 5
+ %el2 = extractelement <vscale x 4 x i1> %a, i32 6
+ %el3 = extractelement <vscale x 4 x i1> %a, i32 7
+ %v0 = insertelement <4 x i1> poison, i1 %el0, i32 0
+ %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
+ %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
+ %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
+ ret <4 x i1> %v3
+}
+
+define <8 x i1> @reshuffle_v8i1_nxv4i1(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: reshuffle_v8i1_nxv4i1:
+; CHECK: // %bb.0:
; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1
; CHECK-NEXT: mov w8, v1.s[1]
; CHECK-NEXT: mov v0.16b, v1.16b
; CHECK-NEXT: mov w9, v1.s[2]
-; CHECK-NEXT: mov v0.h[1], w8
+; CHECK-NEXT: mov z2.s, z1.s[4]
+; CHECK-NEXT: mov v0.b[1], w8
; CHECK-NEXT: mov w8, v1.s[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
+; CHECK-NEXT: mov v0.b[2], w9
+; CHECK-NEXT: mov v0.b[3], w8
+; CHECK-NEXT: fmov w8, s2
+; CHECK-NEXT: mov z2.s, z1.s[5]
+; CHECK-NEXT: mov v0.b[4], w8
+; CHECK-NEXT: fmov w8, s2
+; CHECK-NEXT: mov z2.s, z1.s[6]
+; CHECK-NEXT: mov z1.s, z1.s[7]
+; CHECK-NEXT: mov v0.b[5], w8
+; CHECK-NEXT: fmov w8, s2
+; CHECK-NEXT: mov v0.b[6], w8
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: mov v0.b[7], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%el0 = extractelement <vscale x 4 x i1> %a, i32 0
%el1 = extractelement <vscale x 4 x i1> %a, i32 1
%el2 = extractelement <vscale x 4 x i1> %a, i32 2
%el3 = extractelement <vscale x 4 x i1> %a, i32 3
- %v0 = insertelement <4 x i1> poison, i1 %el0, i32 0
- %v1 = insertelement <4 x i1> %v0, i1 %el1, i32 1
- %v2 = insertelement <4 x i1> %v1, i1 %el2, i32 2
- %v3 = insertelement <4 x i1> %v2, i1 %el3, i32 3
- ret <4 x i1> %v3
+ %el4 = extractelement <vscale x 4 x i1> %a, i32 4
+ %el5 = extractelement <vscale x 4 x i1> %a, i32 5
+ %el6 = extractelement <vscale x 4 x i1> %a, i32 6
+ %el7 = extractelement <vscale x 4 x i1> %a, i32 7
+ %v0 = insertelement <8 x i1> poison, i1 %el0, i32 0
+ %v1 = insertelement <8 x i1> %v0, i1 %el1, i32 1
+ %v2 = insertelement <8 x i1> %v1, i1 %el2, i32 2
+ %v3 = insertelement <8 x i1> %v2, i1 %el3, i32 3
+ %v4 = insertelement <8 x i1> %v3, i1 %el4, i32 4
+ %v5 = insertelement <8 x i1> %v4, i1 %el5, i32 5
+ %v6 = insertelement <8 x i1> %v5, i1 %el6, i32 6
+ %v7 = insertelement <8 x i1> %v6, i1 %el7, i32 7
+ ret <8 x i1> %v7
}
-attributes #0 = { "target-features"="+sve" }
+define <8 x i1> @reshuffle_v8i1_nxv4i1_vscale2(<vscale x 4 x i1> %a) vscale_range(2,2) {
+; CHECK-LABEL: reshuffle_v8i1_nxv4i1_vscale2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %el0 = extractelement <vscale x 4 x i1> %a, i32 0
+ %el1 = extractelement <vscale x 4 x i1> %a, i32 1
+ %el2 = extractelement <vscale x 4 x i1> %a, i32 2
+ %el3 = extractelement <vscale x 4 x i1> %a, i32 3
+ %el4 = extractelement <vscale x 4 x i1> %a, i32 4
+ %el5 = extractelement <vscale x 4 x i1> %a, i32 5
+ %el6 = extractelement <vscale x 4 x i1> %a, i32 6
+ %el7 = extractelement <vscale x 4 x i1> %a, i32 7
+ %v0 = insertelement <8 x i1> poison, i1 %el0, i32 0
+ %v1 = insertelement <8 x i1> %v0, i1 %el1, i32 1
+ %v2 = insertelement <8 x i1> %v1, i1 %el2, i32 2
+ %v3 = insertelement <8 x i1> %v2, i1 %el3, i32 3
+ %v4 = insertelement <8 x i1> %v3, i1 %el4, i32 4
+ %v5 = insertelement <8 x i1> %v4, i1 %el5, i32 5
+ %v6 = insertelement <8 x i1> %v5, i1 %el6, i32 6
+ %v7 = insertelement <8 x i1> %v6, i1 %el7, i32 7
+ ret <8 x i1> %v7
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
index df449f79cc9b5..00536fbfa28d1 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
@@ -168,30 +168,29 @@ define void @test_compressstore_v2i64(ptr %p, <2 x i64> %vec, <2 x i1> %mask) {
define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {
; CHECK-BASE-LABEL: test_compressstore_v8i32:
; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-BASE-NEXT: zip1 v3.8b, v2.8b, v0.8b
+; CHECK-BASE-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-BASE-NEXT: adrp x8, .LCPI11_0
-; CHECK-BASE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-BASE-NEXT: ldr d5, [x8, :lo12:.LCPI11_0]
; CHECK-BASE-NEXT: ptrue p0.s
-; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-BASE-NEXT: ldr d4, [x8, :lo12:.LCPI11_0]
; CHECK-BASE-NEXT: ptrue p1.s, vl4
-; CHECK-BASE-NEXT: shl v4.4h, v3.4h, #15
+; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: shl v3.4h, v2.4h, #15
+; CHECK-BASE-NEXT: cmlt v3.4h, v3.4h, #0
+; CHECK-BASE-NEXT: and v3.8b, v3.8b, v4.8b
+; CHECK-BASE-NEXT: ushll2 v4.4s, v2.8h, #0
; CHECK-BASE-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-BASE-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-BASE-NEXT: cmlt v4.4h, v4.4h, #0
+; CHECK-BASE-NEXT: addv h3, v3.4h
; CHECK-BASE-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-BASE-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-BASE-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-BASE-NEXT: cmpne p2.s, p1/z, z3.s, #0
-; CHECK-BASE-NEXT: addv h4, v4.4h
-; CHECK-BASE-NEXT: cntp x9, p2, p2.s
-; CHECK-BASE-NEXT: compact z0.s, p2, z0.s
-; CHECK-BASE-NEXT: fmov w8, s4
+; CHECK-BASE-NEXT: fmov w8, s3
+; CHECK-BASE-NEXT: shl v3.4s, v4.4s, #31
+; CHECK-BASE-NEXT: cmpne p2.s, p1/z, z2.s, #0
; CHECK-BASE-NEXT: and w8, w8, #0xf
; CHECK-BASE-NEXT: fmov s4, w8
+; CHECK-BASE-NEXT: cntp x9, p2, p2.s
+; CHECK-BASE-NEXT: compact z0.s, p2, z0.s
; CHECK-BASE-NEXT: cnt z4.s, p0/m, z4.s
-; CHECK-BASE-NEXT: cmpne p0.s, p1/z, z2.s, #0
+; CHECK-BASE-NEXT: cmpne p0.s, p1/z, z3.s, #0
; CHECK-BASE-NEXT: whilelo p1.s, xzr, x9
; CHECK-BASE-NEXT: cntp x8, p0, p0.s
; CHECK-BASE-NEXT: compact z1.s, p0, z1.s
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
index bcbf10d848cde..7b0d12e4111e3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
@@ -10,13 +10,8 @@ target triple = "aarch64-unknown-linux-gnu"
define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) {
; CHECK-LABEL: extract_subvector_v8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z0.b, z0.b[4]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: extract_subvector_v8i1:
@@ -43,13 +38,8 @@ define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) {
define <4 x i8> @extract_subvector_v8i8(<8 x i8> %op) {
; CHECK-LABEL: extract_subvector_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z0.b, z0.b[4]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: extract_subvector_v8i8:
@@ -117,9 +107,11 @@ define <2 x i16> @extract_subvector_v4i16(<4 x i16> %op) {
;
; NONEON-NOSVE-LABEL: extract_subvector_v4i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index 8a14f65b3d1f7..c0c9bb5eaf2a8 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -86,22 +86,22 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) {
;
; NONEON-NOSVE-LABEL: sext_v4i3_v4i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-64]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
-; NONEON-NOSVE-NEXT: stp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #24]
-; NONEON-NOSVE-NEXT: ldp w11, w10, [sp, #16]
+; NONEON-NOSVE-NEXT: sub sp, sp, #48
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 48
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrh w11, [sp, #12]
; NONEON-NOSVE-NEXT: sbfx x8, x8, #0, #3
; NONEON-NOSVE-NEXT: sbfx x9, x9, #0, #3
; NONEON-NOSVE-NEXT: sbfx x10, x10, #0, #3
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #48]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT: sbfx x8, x11, #0, #3
-; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #16]
; NONEON-NOSVE-NEXT: stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: add sp, sp, #48
; NONEON-NOSVE-NEXT: ret
%b = sext <4 x i3> %a to <4 x i64>
store <4 x i64> %b, ptr %out
@@ -683,20 +683,18 @@ define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
;
; NONEON-NOSVE-LABEL: sext_v4i8_v4i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-64]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
-; NONEON-NOSVE-NEXT: stp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT: ldrsb x8, [sp, #28]
-; NONEON-NOSVE-NEXT: ldrsb x9, [sp, #24]
-; NONEON-NOSVE-NEXT: ldrsb x10, [sp, #16]
-; NONEON-NOSVE-NEXT: ldrsb x11, [sp, #20]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #48]
-; NONEON-NOSVE-NEXT: stp x10, x11, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: sub sp, sp, #48
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 48
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrsb x8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrsb x9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrsb x10, [sp, #12]
+; NONEON-NOSVE-NEXT: ldrsb x11, [sp, #14]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: stp x10, x11, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #16]
; NONEON-NOSVE-NEXT: stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: add sp, sp, #48
; NONEON-NOSVE-NEXT: ret
%b = sext <4 x i8> %a to <4 x i64>
store <4 x i64>%b, ptr %out
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
index 583dde21d054d..f9e9bb6c09597 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
@@ -2314,21 +2314,11 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; CHECK-LABEL: masked_load_v8f32:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov z1.b, z0.b[3]
-; CHECK-NEXT: mov z2.b, z0.b[2]
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: mov z3.b, z0.b[1]
-; CHECK-NEXT: mov z4.b, z0.b[7]
-; CHECK-NEXT: mov z5.b, z0.b[6]
-; CHECK-NEXT: mov z6.b, z0.b[5]
-; CHECK-NEXT: mov z7.b, z0.b[4]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z2.h, z5.h, z4.h
-; CHECK-NEXT: zip1 z3.h, z7.h, z6.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: zip1 z1.s, z3.s, z2.s
+; CHECK-NEXT: mov x8, #4 // =0x4
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #8
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: lsl z0.s, z0.s, #31
@@ -2338,8 +2328,8 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
; CHECK-NEXT: cmpne p2.s, p0/z, z1.s, #0
; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]
-; CHECK-NEXT: ld1w { z1.s }, p2/z, [x0, x8, lsl #2]
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ld1w { z1.s }, p2/z, [x0, x8, lsl #2]
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z1
; CHECK-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
index 0fa6fca1065c6..19dc5ff64b360 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
@@ -1011,30 +1011,20 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) {
; CHECK-LABEL: masked_store_v8f32:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z4.b, z0.b[4]
-; CHECK-NEXT: mov z5.b, z0.b[3]
-; CHECK-NEXT: mov z6.b, z0.b[2]
-; CHECK-NEXT: mov z7.b, z0.b[1]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z2.h, z4.h, z3.h
-; CHECK-NEXT: zip1 z3.h, z6.h, z5.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z7.h
-; CHECK-NEXT: zip1 z1.s, z2.s, z1.s
-; CHECK-NEXT: zip1 z0.s, z0.s, z3.s
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: mov x8, #4 // =0x4
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #8
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: asr z1.s, z1.s, #31
+; CHECK-NEXT: lsl z1.s, z1.s, #31
; CHECK-NEXT: asr z0.s, z0.s, #31
+; CHECK-NEXT: asr z1.s, z1.s, #31
+; CHECK-NEXT: cmpne p2.s, p0/z, z0.s, #0
; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
; CHECK-NEXT: mov z1.s, #0 // =0x0
-; CHECK-NEXT: cmpne p2.s, p0/z, z0.s, #0
; CHECK-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; CHECK-NEXT: st1w { z1.s }, p2, [x0]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
index 720e0fc77a8c3..572b166649807 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
@@ -11,12 +11,7 @@ define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) {
; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
-; CHECK-NEXT: mov z1.s, z0.s[3]
-; CHECK-NEXT: mov z2.s, z0.s[2]
-; CHECK-NEXT: mov z3.s, z0.s[1]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: ret
%el0 = extractelement <vscale x 4 x i1> %a, i32 0
%el1 = extractelement <vscale x 4 x i1> %a, i32 1
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index df78c2394a942..c87c82c3e3672 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -141,23 +141,20 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v4.16b, v0.16b, v1.16b
; CHECK-NEXT: cmhi v0.16b, v0.16b, v4.16b
; CHECK-NEXT: str q4, [x0]
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: shl v1.4s, v1.4s, #31
; CHECK-NEXT: shl v5.4s, v0.4s, #31
; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: shl v6.4s, v1.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -173,15 +170,12 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v2.8h, v0.8h, v1.8h
; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h
; CHECK-NEXT: str q2, [x0]
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: shl v1.4s, v1.4s, #31
-; CHECK-NEXT: shl v3.4s, v0.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index 0a3d94d8ba952..319348010ad06 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -165,27 +165,24 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
-; CHECK-NEXT: mul v6.16b, v0.16b, v1.16b
+; CHECK-NEXT: mul v7.16b, v0.16b, v1.16b
; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: str q6, [x0]
+; CHECK-NEXT: str q7, [x0]
; CHECK-NEXT: cmtst v2.16b, v2.16b, v2.16b
-; CHECK-NEXT: mov d3, v2.d[1]
-; CHECK-NEXT: zip1 v4.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip1 v5.8b, v3.8b, v0.8b
-; CHECK-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-NEXT: ushll v3.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NEXT: ushll2 v5.4s, v2.8h, #0
+; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v4.4s, v4.4s, #31
-; CHECK-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: shl v5.4s, v5.4s, #31
+; CHECK-NEXT: shl v6.4s, v3.4s, #31
; CHECK-NEXT: shl v2.4s, v2.4s, #31
; CHECK-NEXT: cmlt v0.4s, v4.4s, #0
-; CHECK-NEXT: shl v5.4s, v5.4s, #31
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v5.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -204,15 +201,12 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
; CHECK-NEXT: str q4, [x0]
; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h
-; CHECK-NEXT: xtn v2.8b, v2.8h
-; CHECK-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v3.4s, v3.4s, #31
; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-fcvt.ll b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
index e8a9a830dced4..5af84aeb6943c 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
@@ -18,16 +18,11 @@ define <4 x float> @sitofp_v4i8_float(<4 x i8> %a) {
define <8 x float> @sitofp_v8i8_float(<8 x i8> %a) {
; CHECK-LABEL: sitofp_v8i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NEXT: scvtf v0.4s, v1.4s
-; CHECK-NEXT: scvtf v1.4s, v2.4s
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NEXT: scvtf v0.4s, v0.4s
; CHECK-NEXT: ret
%1 = sitofp <8 x i8> %a to <8 x float>
ret <8 x float> %1
@@ -36,27 +31,16 @@ define <8 x float> @sitofp_v8i8_float(<8 x i8> %a) {
define <16 x float> @sitofp_v16i8_float(<16 x i8> %a) {
; CHECK-LABEL: sitofp_v16i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-NEXT: scvtf v0.4s, v2.4s
-; CHECK-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-NEXT: scvtf v1.4s, v4.4s
-; CHECK-NEXT: scvtf v2.4s, v3.4s
-; CHECK-NEXT: scvtf v3.4s, v5.4s
+; CHECK-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NEXT: scvtf v2.4s, v4.4s
; CHECK-NEXT: ret
%1 = sitofp <16 x i8> %a to <16 x float>
ret <16 x float> %1
@@ -129,14 +113,15 @@ define <4 x float> @uitofp_v4i8_float(<4 x i8> %a) {
define <8 x float> @uitofp_v8i8_float(<8 x i8> %a) {
; CHECK-LABEL: uitofp_v8i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ucvtf v1.4s, v1.4s
; CHECK-NEXT: ret
%1 = uitofp <8 x i8> %a to <8 x float>
ret <8 x float> %1
@@ -145,23 +130,24 @@ define <8 x float> @uitofp_v8i8_float(<8 x i8> %a) {
define <16 x float> @uitofp_v16i8_float(<16 x i8> %a) {
; CHECK-LABEL: uitofp_v16i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NEXT: bic v3.4h, #255, lsl #8
; CHECK-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-NEXT: ucvtf v1.4s, v5.4s
; CHECK-NEXT: ret
%1 = uitofp <16 x i8> %a to <16 x float>
ret <16 x float> %1
@@ -242,31 +228,27 @@ define <4 x double> @sitofp_v4i8_double(<4 x i8> %a) {
define <8 x double> @sitofp_v8i8_double(<8 x i8> %a) {
; CHECK-LABEL: sitofp_v8i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov b1, v0.b[0]
-; CHECK-NEXT: mov b2, v0.b[2]
-; CHECK-NEXT: mov b3, v0.b[4]
-; CHECK-NEXT: mov b4, v0.b[6]
-; CHECK-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v1.2s, v1.2s, #24
; CHECK-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NEXT: shl v3.2s, v3.2s, #24
; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-NEXT: scvtf v2.2d, v1.2d
+; CHECK-NEXT: scvtf v3.2d, v4.2d
+; CHECK-NEXT: scvtf v1.2d, v5.2d
; CHECK-NEXT: ret
%1 = sitofp <8 x i8> %a to <8 x double>
ret <8 x double> %1
@@ -275,55 +257,48 @@ define <8 x double> @sitofp_v8i8_double(<8 x i8> %a) {
define <16 x double> @sitofp_v16i8_double(<16 x i8> %a) {
; CHECK-LABEL: sitofp_v16i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: mov b2, v0.b[0]
-; CHECK-NEXT: mov b3, v0.b[2]
-; CHECK-NEXT: mov b4, v0.b[4]
-; CHECK-NEXT: mov b5, v0.b[6]
-; CHECK-NEXT: mov b6, v1.b[0]
-; CHECK-NEXT: mov b7, v1.b[2]
-; CHECK-NEXT: mov b16, v1.b[4]
-; CHECK-NEXT: mov b17, v1.b[6]
-; CHECK-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d4, v2.d[1]
+; CHECK-NEXT: mov d5, v1.d[1]
+; CHECK-NEXT: mov d6, v3.d[1]
+; CHECK-NEXT: mov d7, v0.d[1]
+; CHECK-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NEXT: shl v7.2s, v7.2s, #24
; CHECK-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NEXT: scvtf v0.2d, v1.2d
+; CHECK-NEXT: scvtf v4.2d, v16.2d
+; CHECK-NEXT: scvtf v6.2d, v18.2d
+; CHECK-NEXT: scvtf v3.2d, v17.2d
+; CHECK-NEXT: scvtf v1.2d, v5.2d
+; CHECK-NEXT: scvtf v7.2d, v19.2d
+; CHECK-NEXT: scvtf v5.2d, v20.2d
; CHECK-NEXT: ret
%1 = sitofp <16 x i8> %a to <16 x double>
ret <16 x double> %1
@@ -395,28 +370,24 @@ define <4 x double> @uitofp_v4i8_double(<4 x i8> %a) {
define <8 x double> @uitofp_v8i8_double(<8 x i8> %a) {
; CHECK-LABEL: uitofp_v8i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov b2, v0.b[0]
-; CHECK-NEXT: mov b3, v0.b[2]
-; CHECK-NEXT: mov b4, v0.b[4]
-; CHECK-NEXT: mov b5, v0.b[6]
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d3, v2.d[1]
+; CHECK-NEXT: mov d4, v0.d[1]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NEXT: ucvtf v1.2d, v1.2d
; CHECK-NEXT: ret
%1 = uitofp <8 x i8> %a to <8 x double>
ret <8 x double> %1
@@ -425,48 +396,41 @@ define <8 x double> @uitofp_v8i8_double(<8 x i8> %a) {
define <16 x double> @uitofp_v16i8_double(<16 x i8> %a) {
; CHECK-LABEL: uitofp_v16i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov b3, v0.b[0]
-; CHECK-NEXT: mov b4, v0.b[2]
-; CHECK-NEXT: mov b5, v0.b[4]
-; CHECK-NEXT: mov b6, v0.b[6]
+; CHECK-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NEXT: mov b7, v2.b[0]
-; CHECK-NEXT: mov b16, v2.b[2]
-; CHECK-NEXT: mov b17, v2.b[4]
-; CHECK-NEXT: mov b18, v2.b[6]
-; CHECK-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d5, v3.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: mov d7, v4.d[1]
+; CHECK-NEXT: mov d16, v0.d[1]
+; CHECK-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NEXT: ucvtf v5.2d, v19.2d
; CHECK-NEXT: ret
%1 = uitofp <16 x i8> %a to <16 x double>
ret <16 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index b066abd0a19d5..45b832ed1cff8 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1262,44 +1262,54 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
; CHECK-NEXT: ldr x9, [x0, x8]
; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: cmp x8, #128
-; CHECK-NEXT: ubfx x12, x9, #48, #4
-; CHECK-NEXT: ubfx x10, x9, #52, #4
-; CHECK-NEXT: ubfx x13, x9, #32, #4
-; CHECK-NEXT: ubfx w15, w9, #16, #4
+; CHECK-NEXT: ubfx x10, x9, #32, #4
+; CHECK-NEXT: and w12, w9, #0xf
; CHECK-NEXT: ubfx x11, x9, #36, #4
-; CHECK-NEXT: ubfx w14, w9, #20, #4
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: fmov s1, w13
-; CHECK-NEXT: ubfx w12, w9, #4, #4
-; CHECK-NEXT: fmov s2, w15
-; CHECK-NEXT: mov.h v0[1], w10
-; CHECK-NEXT: and w10, w9, #0xf
-; CHECK-NEXT: mov.h v1[1], w11
-; CHECK-NEXT: fmov s3, w10
+; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #4, #4
+; CHECK-NEXT: mov.b v1[1], w10
+; CHECK-NEXT: ubfx w10, w9, #8, #4
+; CHECK-NEXT: mov.b v0[1], w11
+; CHECK-NEXT: ubfx x11, x9, #40, #4
+; CHECK-NEXT: mov.b v1[2], w10
+; CHECK-NEXT: ubfx w10, w9, #12, #4
+; CHECK-NEXT: mov.b v0[2], w11
+; CHECK-NEXT: ubfx x11, x9, #44, #4
+; CHECK-NEXT: mov.b v1[3], w10
+; CHECK-NEXT: ubfx w10, w9, #16, #4
+; CHECK-NEXT: mov.b v0[3], w11
+; CHECK-NEXT: ubfx x11, x9, #48, #4
+; CHECK-NEXT: mov.b v1[4], w10
+; CHECK-NEXT: ubfx w10, w9, #20, #4
+; CHECK-NEXT: mov.b v0[4], w11
+; CHECK-NEXT: ubfx x11, x9, #52, #4
+; CHECK-NEXT: mov.b v1[5], w10
+; CHECK-NEXT: ubfx w10, w9, #24, #4
+; CHECK-NEXT: mov.b v0[5], w11
; CHECK-NEXT: ubfx x11, x9, #56, #4
-; CHECK-NEXT: mov.h v2[1], w14
-; CHECK-NEXT: ubfx x10, x9, #40, #4
-; CHECK-NEXT: mov.h v3[1], w12
-; CHECK-NEXT: ubfx w12, w9, #24, #4
-; CHECK-NEXT: mov.h v0[2], w11
-; CHECK-NEXT: ubfx w11, w9, #8, #4
-; CHECK-NEXT: mov.h v1[2], w10
-; CHECK-NEXT: lsr x10, x9, #60
-; CHECK-NEXT: mov.h v2[2], w12
-; CHECK-NEXT: ubfx x12, x9, #44, #4
-; CHECK-NEXT: mov.h v3[2], w11
-; CHECK-NEXT: lsr w11, w9, #28
-; CHECK-NEXT: ubfx w9, w9, #12, #4
-; CHECK-NEXT: mov.h v0[3], w10
-; CHECK-NEXT: mov.h v1[3], w12
-; CHECK-NEXT: mov.h v2[3], w11
-; CHECK-NEXT: mov.h v3[3], w9
-; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: mov.b v1[6], w10
+; CHECK-NEXT: mov.b v0[6], w11
+; CHECK-NEXT: lsr x11, x9, #60
+; CHECK-NEXT: lsr w9, w9, #28
+; CHECK-NEXT: mov.b v1[7], w9
+; CHECK-NEXT: mov.b v0[7], w11
+; CHECK-NEXT: ushll.8h v1, v1, #0
+; CHECK-NEXT: ushll.8h v0, v0, #0
+; CHECK-NEXT: mov d3, v1[1]
+; CHECK-NEXT: ; kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: mov d2, v0[1]
+; CHECK-NEXT: ; kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: bic.4h v3, #255, lsl #8
; CHECK-NEXT: ushll.4s v1, v1, #0
-; CHECK-NEXT: ushll.4s v2, v2, #0
+; CHECK-NEXT: bic.4h v2, #255, lsl #8
+; CHECK-NEXT: ushll.4s v0, v0, #0
; CHECK-NEXT: ushll.4s v3, v3, #0
-; CHECK-NEXT: stp q1, q0, [x1, #32]
-; CHECK-NEXT: stp q3, q2, [x1], #64
+; CHECK-NEXT: ushll.4s v2, v2, #0
+; CHECK-NEXT: stp q0, q2, [x1, #32]
+; CHECK-NEXT: stp q1, q3, [x1], #64
; CHECK-NEXT: b.ne LBB13_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: ret
@@ -1312,51 +1322,60 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
; CHECK-BE-NEXT: ldr x9, [x0, x8]
; CHECK-BE-NEXT: add x8, x8, #16
; CHECK-BE-NEXT: cmp x8, #128
-; CHECK-BE-NEXT: lsr x13, x9, #60
-; CHECK-BE-NEXT: ubfx w12, w9, #12, #4
-; CHECK-BE-NEXT: lsr w14, w9, #28
-; CHECK-BE-NEXT: ubfx x16, x9, #56, #4
-; CHECK-BE-NEXT: ubfx x17, x9, #44, #4
-; CHECK-BE-NEXT: ubfx w10, w9, #8, #4
-; CHECK-BE-NEXT: fmov s0, w13
-; CHECK-BE-NEXT: fmov s1, w12
-; CHECK-BE-NEXT: fmov s2, w14
+; CHECK-BE-NEXT: lsr w10, w9, #28
+; CHECK-BE-NEXT: lsr x12, x9, #60
; CHECK-BE-NEXT: ubfx w11, w9, #24, #4
-; CHECK-BE-NEXT: fmov s3, w17
-; CHECK-BE-NEXT: ubfx x15, x9, #40, #4
-; CHECK-BE-NEXT: ubfx w12, w9, #20, #4
-; CHECK-BE-NEXT: ubfx x13, x9, #36, #4
-; CHECK-BE-NEXT: mov v0.h[1], w16
-; CHECK-BE-NEXT: mov v1.h[1], w10
-; CHECK-BE-NEXT: mov v2.h[1], w11
+; CHECK-BE-NEXT: ubfx x13, x9, #56, #4
+; CHECK-BE-NEXT: fmov s0, w10
+; CHECK-BE-NEXT: fmov s1, w12
+; CHECK-BE-NEXT: ubfx w10, w9, #20, #4
+; CHECK-BE-NEXT: mov v0.b[1], w11
+; CHECK-BE-NEXT: mov v1.b[1], w13
; CHECK-BE-NEXT: ubfx x11, x9, #52, #4
+; CHECK-BE-NEXT: mov v0.b[2], w10
+; CHECK-BE-NEXT: mov v1.b[2], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #16, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #48, #4
+; CHECK-BE-NEXT: mov v0.b[3], w10
+; CHECK-BE-NEXT: mov v1.b[3], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #12, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #44, #4
+; CHECK-BE-NEXT: mov v0.b[4], w10
+; CHECK-BE-NEXT: mov v1.b[4], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #8, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #40, #4
+; CHECK-BE-NEXT: mov v0.b[5], w10
+; CHECK-BE-NEXT: mov v1.b[5], w11
; CHECK-BE-NEXT: ubfx w10, w9, #4, #4
-; CHECK-BE-NEXT: mov v3.h[1], w15
-; CHECK-BE-NEXT: mov v0.h[2], w11
-; CHECK-BE-NEXT: mov v1.h[2], w10
-; CHECK-BE-NEXT: ubfx x10, x9, #48, #4
-; CHECK-BE-NEXT: mov v2.h[2], w12
-; CHECK-BE-NEXT: mov v3.h[2], w13
-; CHECK-BE-NEXT: and w11, w9, #0xf
-; CHECK-BE-NEXT: ubfx w12, w9, #16, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #36, #4
+; CHECK-BE-NEXT: mov v0.b[6], w10
+; CHECK-BE-NEXT: mov v1.b[6], w11
+; CHECK-BE-NEXT: and w10, w9, #0xf
; CHECK-BE-NEXT: ubfx x9, x9, #32, #4
-; CHECK-BE-NEXT: mov v0.h[3], w10
-; CHECK-BE-NEXT: mov v1.h[3], w11
-; CHECK-BE-NEXT: mov v2.h[3], w12
-; CHECK-BE-NEXT: mov v3.h[3], w9
-; CHECK-BE-NEXT: add x9, x1, #48
+; CHECK-BE-NEXT: mov v0.b[7], w10
+; CHECK-BE-NEXT: mov v1.b[7], w9
+; CHECK-BE-NEXT: add x9, x1, #32
+; CHECK-BE-NEXT: add x10, x1, #48
+; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BE-NEXT: mov d2, v0.d[1]
+; CHECK-BE-NEXT: mov d3, v1.d[1]
+; CHECK-BE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-BE-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-BE-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-BE-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-BE-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-BE-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-BE-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-BE-NEXT: st1 { v1.4s }, [x9]
-; CHECK-BE-NEXT: add x9, x1, #32
-; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
+; CHECK-BE-NEXT: st1 { v0.4s }, [x9]
; CHECK-BE-NEXT: add x9, x1, #16
-; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
-; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT: st1 { v1.4s }, [x1]
; CHECK-BE-NEXT: add x1, x1, #64
+; CHECK-BE-NEXT: st1 { v2.4s }, [x10]
+; CHECK-BE-NEXT: st1 { v3.4s }, [x9]
; CHECK-BE-NEXT: b.ne .LBB13_1
; CHECK-BE-NEXT: // %bb.2: // %exit
; CHECK-BE-NEXT: ret
More information about the llvm-commits
mailing list