[llvm] [AArch64] Optimize lowering of extending BUILD_VECTOR(EXTRACT_VECTOR_ELT()) (PR #212156)
Usman Nadeem via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 11:48:41 PDT 2026
https://github.com/UsmanNadeem updated https://github.com/llvm/llvm-project/pull/212156
>From 8c44cbc1b5fb2b5d03d629782bd833b383fc20e2 Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 17:41:29 -0700
Subject: [PATCH 1/5] [AArch64] Optimize lowering of extending BUILD_VECTOR
Lower them to EXTRACT_SUBVECTOR in more cases, including for fixed vector extracts of predicates.
Change-Id: Ia4fa4c7d8cf5fa7abe1318d00e360a895cdbd109
---
.../Target/AArch64/AArch64ISelLowering.cpp | 185 ++++++--
llvm/test/CodeGen/AArch64/aarch64-load-ext.ll | 9 +-
.../aarch64-neon-vector-insert-uaddlv.ll | 6 +-
llvm/test/CodeGen/AArch64/add.ll | 4 +-
llvm/test/CodeGen/AArch64/andorxor.ll | 6 +-
llvm/test/CodeGen/AArch64/bitcast-extend.ll | 20 +-
.../CodeGen/AArch64/combine-storetomstore.ll | 105 +++--
llvm/test/CodeGen/AArch64/ctlz.ll | 4 +-
llvm/test/CodeGen/AArch64/extbinopload.ll | 4 +-
llvm/test/CodeGen/AArch64/itofp.ll | 408 ++++++++----------
llvm/test/CodeGen/AArch64/mul.ll | 6 +-
llvm/test/CodeGen/AArch64/sub.ll | 4 +-
.../sve-extract-fixed-from-scalable-vector.ll | 12 +-
.../AArch64/sve-extract-fixed-vector.ll | 49 +--
.../AArch64/sve-fixed-length-masked-gather.ll | 18 +-
.../sve-fixed-length-masked-scatter.ll | 25 +-
.../AArch64/sve-fixed-length-reshuffle.ll | 11 +-
.../AArch64/sve-masked-compressstore.ll | 31 +-
...ing-mode-fixed-length-extract-subvector.ll | 24 +-
...streaming-mode-fixed-length-int-extends.ll | 46 +-
...streaming-mode-fixed-length-masked-load.ll | 20 +-
...treaming-mode-fixed-length-masked-store.ll | 26 +-
...e-streaming-mode-fixed-length-reshuffle.ll | 7 +-
llvm/test/CodeGen/AArch64/vec_uaddo.ll | 11 +-
llvm/test/CodeGen/AArch64/vec_umulo.ll | 9 +-
llvm/test/CodeGen/AArch64/vector-fcvt.ll | 86 ++--
26 files changed, 530 insertions(+), 606 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e9fdaaab967af..b81ef10a71bc2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23660,36 +23660,167 @@ static SDValue performBuildVectorCombine(SDNode *N,
// (extract_elt_iXX_to_i32 vec Idx+1))
// => (extract_subvector (anyext_iXX_to_i32 vec) Idx)
- // For now, only consider the v2i32 case, which arises as a result of
- // legalization.
- if (VT != MVT::v2i32)
- return SDValue();
-
- SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
- // Reminder, EXTRACT_VECTOR_ELT has the effect of any-extending to its VT.
- if (Elt0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- Elt1->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- // Constant index.
- isa<ConstantSDNode>(Elt0->getOperand(1)) &&
- isa<ConstantSDNode>(Elt1->getOperand(1)) &&
- // Both EXTRACT_VECTOR_ELT from same vector...
- Elt0->getOperand(0) == Elt1->getOperand(0) &&
- // ... and contiguous. First element's index +1 == second element's index.
- Elt0->getConstantOperandVal(1) + 1 == Elt1->getConstantOperandVal(1) &&
- // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
- // ResultType's known minimum vector length.
- Elt0->getConstantOperandVal(1) % VT.getVectorMinNumElements() == 0) {
- SDValue VecToExtend = Elt0->getOperand(0);
- EVT ExtVT = VecToExtend.getValueType().changeVectorElementType(
- *DAG.getContext(), MVT::i32);
- if (!DAG.getTargetLoweringInfo().isTypeLegal(ExtVT))
- return SDValue();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ SDValue Op0 = N->getOperand(0);
+
+ auto IsValidBV = [&]() {
+ if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return false;
+ if (!isa<ConstantSDNode>(Op0.getOperand(1)))
+ return false;
+ uint64_t ExtractIdx = Op0.getConstantOperandVal(1);
+ // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
+ // ResultType's known minimum vector length.
+ if (ExtractIdx % VT.getVectorMinNumElements() != 0)
+ return false;
+ // Check that the vector we are extracting from is the same in all extracts
+ // and the indices are sequential.
+ for (const SDValue &Op : drop_begin(N->op_values())) {
+ if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return false;
+ if (Op.getOperand(0) != Op0.getOperand(0))
+ return false;
+ if (!isa<ConstantSDNode>(Op.getOperand(1)))
+ return false;
+ if (Op.getConstantOperandVal(1) != ++ExtractIdx)
+ return false;
+ }
+ return true;
+ };
+
+ if (!IsValidBV())
+ return SDValue();
+
+ SDValue VecToExtract = Op0.getOperand(0);
+ EVT PreExtractVT = VecToExtract.getValueType();
+ uint64_t ExtractIdx = Op0.getConstantOperandVal(1);
- SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend);
- return DAG.getExtractSubvector(DL, MVT::v2i32, Ext,
- Elt0->getConstantOperandVal(1));
+ // Scalable predicate to fixed vector extracts.
+ if (PreExtractVT.isScalableVectorOf(MVT::i1) && !DCI.isBeforeLegalize() &&
+ VT.isFixedLengthVector()) {
+ EVT FullIntVT;
+ switch (PreExtractVT.getVectorMinNumElements()) {
+ default:
+ llvm_unreachable("Unexpected element type for SVE predicate.");
+ case 16:
+ FullIntVT = MVT::nxv16i8;
+ break;
+ case 8:
+ FullIntVT = MVT::nxv8i16;
+ break;
+ case 4:
+ FullIntVT = MVT::nxv4i32;
+ break;
+ case 2:
+ FullIntVT = MVT::nxv2i64;
+ break;
+ }
+ // We want to extend the predicate to the corresponding int type with the
+ // same number of elements. Then we extract a fixed vector of the same type
+ // and then cast it to an appropirate width.
+ //
+ // If the extended_pred's type is the default SVE container for the desired
+ // fixed type then we can simply do an extract. For example, for nxv8i1 ->
+ // v4i16 we get:
+ //
+ // nxv8i1 -> nxv8i16 -> extract v4i16.
+ //
+ // If the fixed vector extract of the predicate already has the desired
+ // element count we just need to any_ext/trunc(extended_pred) to the right
+ // width. For example, for nxv4i1 -> v4i16, we get:
+ //
+ // nxv4i1 -> nxv4i32 -> extract v4i32 -> v4i16.
+ //
+ // Finally, for something like nxv16i1 -> v4i32, we do:
+ //
+ // nxv16i1 -> nxv16i8 -unpack-> nxv4i32 -> extract v4i32.
+ // the unpack is till we get to the container type of the result.
+
+ EVT ExtractFixedVT = VT.changeVectorElementType(
+ *DAG.getContext(), FullIntVT.getVectorElementType());
+ EVT ResultContainerTy = getContainerForFixedLengthVector(DAG, VT);
+ SDValue ExtendedPred = DAG.getAnyExtOrTrunc(VecToExtract, DL, FullIntVT);
+
+ // Same container type, just extract.
+ if (ResultContainerTy == FullIntVT)
+ return DAG.getExtractSubvector(DL, VT, ExtendedPred, ExtractIdx);
+
+ // Same number of elements but different type, extract and then trunc/ext.
+ if (FullIntVT.getVectorMinNumElements() == VT.getVectorMinNumElements()) {
+ if (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(ExtractFixedVT.getScalarSizeInBits() >
+ VT.getScalarSizeInBits()
+ ? ISD::ANY_EXTEND
+ : ISD::TRUNCATE,
+ VT))
+ return SDValue();
+ SDValue ExtractFixed =
+ DAG.getExtractSubvector(DL, ExtractFixedVT, ExtendedPred, ExtractIdx);
+ return DAG.getAnyExtOrTrunc(ExtractFixed, DL, VT);
+ } else {
+ unsigned UnpackOpcode = ISD::ANY_EXTEND_VECTOR_INREG;
+ // Unpack extended_pred till we get to the container type. Adjust the
+ // ExtractIdx accordingly.
+ switch (FullIntVT.getSimpleVT().SimpleTy) {
+ default:
+ return SDValue();
+ case MVT::nxv16i8:
+ if (ExtractIdx > 7) {
+ UnpackOpcode = AArch64ISD::UUNPKHI;
+ ExtractIdx -= 8;
+ } else
+ UnpackOpcode = ISD::ANY_EXTEND_VECTOR_INREG;
+ ExtendedPred =
+ DAG.getNode(UnpackOpcode, DL, MVT::nxv8i16, ExtendedPred);
+ if (ResultContainerTy == MVT::nxv8i16)
+ break;
+ [[fallthrough]];
+ case MVT::nxv8i16:
+ if (ExtractIdx > 3) {
+ UnpackOpcode = AArch64ISD::UUNPKHI;
+ ExtractIdx -= 4;
+ } else
+ UnpackOpcode = ISD::ANY_EXTEND_VECTOR_INREG;
+ ExtendedPred =
+ DAG.getNode(UnpackOpcode, DL, MVT::nxv4i32, ExtendedPred);
+ if (ResultContainerTy == MVT::nxv4i32)
+ break;
+ [[fallthrough]];
+ case MVT::nxv4i32:
+ if (ExtractIdx > 1) {
+ UnpackOpcode = AArch64ISD::UUNPKHI;
+ ExtractIdx -= 2;
+ } else
+ UnpackOpcode = ISD::ANY_EXTEND_VECTOR_INREG;
+ ExtendedPred =
+ DAG.getNode(UnpackOpcode, DL, MVT::nxv2i64, ExtendedPred);
+ assert(ResultContainerTy == MVT::nxv2i64 && "Unexpected result type!");
+ break;
+ }
+ return DAG.getExtractSubvector(DL, VT, ExtendedPred, ExtractIdx);
+ }
}
+ // For now, only consider the v2i32 and v4i16 cases, which arises as a result
+ // of legalization.
+ if (VT != MVT::v2i32 && VT != MVT::v4i16)
+ return SDValue();
+
+ // Not sure if we will ever hit the non-extend case.
+ if (PreExtractVT.getScalarSizeInBits() >= VT.getScalarSizeInBits())
+ return SDValue();
+
+ EVT ExtVT = PreExtractVT.changeVectorElementType(*DAG.getContext(),
+ VT.getVectorElementType());
+
+ if (!TLI.isTypeLegal(ExtVT) && !DCI.isBeforeLegalize())
+ return SDValue();
+
+ if (DCI.isBeforeLegalizeOps() ||
+ (TLI.isOperationLegal(ISD::EXTRACT_SUBVECTOR, VT) &&
+ TLI.isOperationLegal(ISD::ANY_EXTEND, ExtVT)))
+ return DAG.getExtractSubvector(
+ DL, VT, DAG.getAnyExtOrTrunc(VecToExtract, DL, ExtVT), ExtractIdx);
return SDValue();
}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll b/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
index 0ef2b31d00daa..b910eb6423f35 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-load-ext.ll
@@ -271,18 +271,15 @@ define <3 x i16> @fsext_v3i16(ptr %a) {
; CHECK-LE-LABEL: fsext_v3i16:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: ldr s0, [x0]
-; CHECK-LE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: fsext_v3i16:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: ldr s0, [x0]
; CHECK-BE-NEXT: rev32 v0.8b, v0.8b
-; CHECK-BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
; CHECK-BE-NEXT: ret
%x = load <3 x i8>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d552397..bbb9e98a39e31 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -281,7 +281,8 @@ define void @insert_vec_v16i8_uaddlv_from_v8i8(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.8b h1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.h v2[0], v1[0]
+; CHECK-NEXT: mov.b v2[0], v1[0]
+; CHECK-NEXT: zip1.8b v2, v2, v2
; CHECK-NEXT: bic.4h v2, #255, lsl #8
; CHECK-NEXT: ushll.4s v2, v2, #0
; CHECK-NEXT: ucvtf.4s v2, v2
@@ -457,7 +458,8 @@ define void @insert_vec_v16i8_uaddlv_from_v4i32(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4s d0, v0
; CHECK-NEXT: stp q2, q2, [x0, #32]
-; CHECK-NEXT: mov.h v1[0], v0[0]
+; CHECK-NEXT: mov.b v1[0], v0[0]
+; CHECK-NEXT: zip1.8b v1, v1, v1
; CHECK-NEXT: bic.4h v1, #255, lsl #8
; CHECK-NEXT: ushll.4s v1, v1, #0
; CHECK-NEXT: ucvtf.4s v1, v1
diff --git a/llvm/test/CodeGen/AArch64/add.ll b/llvm/test/CodeGen/AArch64/add.ll
index f71c58f16bf36..833a00d44006c 100644
--- a/llvm/test/CodeGen/AArch64/add.ll
+++ b/llvm/test/CodeGen/AArch64/add.ll
@@ -94,9 +94,7 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: add v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
diff --git a/llvm/test/CodeGen/AArch64/andorxor.ll b/llvm/test/CodeGen/AArch64/andorxor.ll
index 6e1f7710b4cd8..1859ac348b0ab 100644
--- a/llvm/test/CodeGen/AArch64/andorxor.ll
+++ b/llvm/test/CodeGen/AArch64/andorxor.ll
@@ -289,7 +289,7 @@ define void @and_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
@@ -340,7 +340,7 @@ define void @or_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
@@ -391,7 +391,7 @@ define void @xor_v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: eor v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
diff --git a/llvm/test/CodeGen/AArch64/bitcast-extend.ll b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
index 23cfe5a2f399f..60ef112c979f3 100644
--- a/llvm/test/CodeGen/AArch64/bitcast-extend.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
@@ -98,20 +98,12 @@ define <4 x i64> @z_i32_v4i64(i32 %x) {
}
define <4 x i16> @s_i32_v4i16(i32 %x) {
-; CHECK-SD-LABEL: s_i32_v4i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: s_i32_v4i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: s_i32_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
%b = bitcast i32 %x to <4 x i8>
%e = sext <4 x i8> %b to <4 x i16>
ret <4 x i16> %e
diff --git a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
index 35e1f3ea3020c..3b0573b7e5c28 100644
--- a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
@@ -160,13 +160,12 @@ define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -243,13 +242,12 @@ define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %
define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -727,13 +725,12 @@ define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
@@ -814,18 +811,17 @@ define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask)
define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_load:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldr q4, [x0]
-; SVE-NEXT: ldr q5, [x0, #16]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldr q4, [x0, #16]
+; SVE-NEXT: ldr q5, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: stp q0, q1, [x0]
; SVE-NEXT: ret
%load = load volatile <8 x i32>, ptr %ptr, align 32
@@ -837,19 +833,18 @@ define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_volatile_store:
; SVE: // %bb.0:
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: ldp q4, q5, [x0]
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
+; SVE-NEXT: ldp q5, q4, [x0]
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmlt v3.4s, v3.4s, #0
; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: bif v0.16b, v4.16b, v3.16b
-; SVE-NEXT: bif v1.16b, v5.16b, v2.16b
-; SVE-NEXT: str q0, [x0]
+; SVE-NEXT: bif v1.16b, v4.16b, v3.16b
+; SVE-NEXT: bif v0.16b, v5.16b, v2.16b
; SVE-NEXT: str q1, [x0, #16]
+; SVE-NEXT: str q0, [x0]
; SVE-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 32
%sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
@@ -874,21 +869,20 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; SVE-NEXT: movi v1.2d, #0000000000000000
; SVE-NEXT: stp q0, q0, [x0]
; SVE-NEXT: bl use_vec
-; SVE-NEXT: zip2 v0.8b, v8.8b, v0.8b
+; SVE-NEXT: ushll v0.8h, v8.8b, #0
; SVE-NEXT: ldp q3, q2, [sp, #16] // 32-byte Folded Reload
-; SVE-NEXT: zip1 v1.8b, v8.8b, v0.8b
-; SVE-NEXT: ushll v0.4s, v0.4h, #0
; SVE-NEXT: ldr d8, [sp, #64] // 8-byte Reload
+; SVE-NEXT: ushll2 v1.4s, v0.8h, #0
+; SVE-NEXT: ushll v0.4s, v0.4h, #0
+; SVE-NEXT: shl v1.4s, v1.4s, #31
; SVE-NEXT: shl v0.4s, v0.4s, #31
-; SVE-NEXT: ushll v1.4s, v1.4h, #0
+; SVE-NEXT: cmlt v1.4s, v1.4s, #0
; SVE-NEXT: cmlt v0.4s, v0.4s, #0
-; SVE-NEXT: shl v1.4s, v1.4s, #31
-; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
; SVE-NEXT: ldr q2, [sp, #48] // 16-byte Reload
; SVE-NEXT: ldr q3, [sp] // 16-byte Reload
-; SVE-NEXT: cmlt v1.4s, v1.4s, #0
-; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b
-; SVE-NEXT: stp q1, q0, [x19]
+; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b
+; SVE-NEXT: stp q0, q1, [x19]
; SVE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload
; SVE-NEXT: add sp, sp, #96
; SVE-NEXT: ret
@@ -905,16 +899,14 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {
; SVE-LABEL: test_masked_store_multiple_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v6.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v7.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll v5.8h, v5.8b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v5.4s, v5.4h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
@@ -924,10 +916,10 @@ define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %p
; SVE-NEXT: cmlt v6.4s, v6.4s, #0
; SVE-NEXT: cmpne p1.s, p0/z, z7.s, #0
; SVE-NEXT: cmlt v5.4s, v5.4s, #0
-; SVE-NEXT: ldp q7, q16, [x1]
+; SVE-NEXT: ldp q16, q7, [x1]
; SVE-NEXT: cmpne p2.s, p0/z, z4.s, #0
-; SVE-NEXT: bif v2.16b, v7.16b, v6.16b
-; SVE-NEXT: bif v3.16b, v16.16b, v5.16b
+; SVE-NEXT: bif v3.16b, v7.16b, v6.16b
+; SVE-NEXT: bif v2.16b, v16.16b, v5.16b
; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: st1w { z0.s }, p2, [x0]
; SVE-NEXT: stp q2, q3, [x1]
@@ -1066,21 +1058,20 @@ define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i
define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; SVE-NEXT: add x8, x0, #1
+; SVE-NEXT: ushll v2.8h, v2.8b, #0
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: add x9, x0, #17
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: add x9, x0, #1
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: ushll v3.4s, v3.4h, #0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v3.4s, v2.8h, #0
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: shl v3.4s, v3.4s, #31
; SVE-NEXT: shl v2.4s, v2.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #0
; SVE-NEXT: cmpne p2.s, p0/z, z2.s, #0
-; SVE-NEXT: st1w { z0.s }, p1, [x8]
-; SVE-NEXT: st1w { z1.s }, p2, [x9]
+; SVE-NEXT: st1w { z1.s }, p1, [x8]
+; SVE-NEXT: st1w { z0.s }, p2, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i32 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
index 968fd6d7c9813..6f89dfafd9664 100644
--- a/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -38,9 +38,9 @@ define void @v3i8(ptr %p1) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: movi v0.4h, #8
; CHECK-SD-NEXT: ldr s1, [x0]
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: movi v0.4h, #8
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
; CHECK-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT: clz v1.4h, v1.4h
; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll
index d18cff51c6101..6178fd1ab8f15 100644
--- a/llvm/test/CodeGen/AArch64/extbinopload.ll
+++ b/llvm/test/CodeGen/AArch64/extbinopload.ll
@@ -1365,11 +1365,11 @@ define <4 x i32> @atomic(ptr %p) {
; CHECK-LABEL: atomic:
; CHECK: // %bb.0:
; CHECK-NEXT: ldar w8, [x0]
-; CHECK-NEXT: movi v0.2d, #0x0000ff000000ff
; CHECK-NEXT: ldr s1, [x0, #4]
+; CHECK-NEXT: movi v0.2d, #0x0000ff000000ff
; CHECK-NEXT: fmov s2, w8
; CHECK-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NEXT: zip1 v2.8b, v2.8b, v0.8b
+; CHECK-NEXT: zip1 v2.8b, v2.8b, v2.8b
; CHECK-NEXT: ushll v1.4s, v1.4h, #3
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: and v0.16b, v2.16b, v0.16b
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index c61b38d73c29d..e966614eb7238 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -4763,30 +4763,20 @@ entry:
define <8 x float> @stofp_v8i8_v8f32(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v1.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v0.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.4s, v1.4s
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v0.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f32:
@@ -4814,26 +4804,28 @@ entry:
define <8 x float> @utofp_v8i8_v8f32(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v1.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v1.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f32:
@@ -4861,52 +4853,30 @@ entry:
define <16 x float> @stofp_v16i8_v16f32(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v2.4s
-; CHECK-NOFP16-SD-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v3.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v4.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-FP16-SD-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-FP16-SD-NEXT: scvtf v0.4s, v2.4s
-; CHECK-FP16-SD-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v4.4s
-; CHECK-FP16-SD-NEXT: scvtf v2.4s, v3.4s
-; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT: scvtf v2.4s, v4.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f32:
@@ -4944,44 +4914,46 @@ entry:
define <16 x float> @utofp_v16i8_v16f32(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v5.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v5.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f32:
@@ -5019,94 +4991,50 @@ entry:
define <32 x float> @stofp_v32i8_v32f32(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v4.4h, v4.4h, #8
-; CHECK-NOFP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v4.4h, v4.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v5.4h, v5.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v6.4h, v6.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v7.4h, v7.4h, #8
-; CHECK-NOFP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v4.4s, v4.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshr v5.4h, v5.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v6.4h, v6.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v7.4h, v7.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NOFP16-SD-NEXT: sshll v16.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v4.4s
-; CHECK-NOFP16-SD-NEXT: sshll v5.4s, v5.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v6.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v17.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v16.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: sshll2 v17.4s, v3.8h, #0
; CHECK-NOFP16-SD-NEXT: sshll v18.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-NOFP16-SD-NEXT: sshll v19.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v5.4s
; CHECK-NOFP16-SD-NEXT: scvtf v2.4s, v6.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v3.4s, v17.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v5.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v18.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v7.4s
-; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v4.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v7.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v5.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: scvtf v6.4s, v18.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v4.4h, v4.4h, #8
-; CHECK-FP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v4.4h, v4.4h, #8
-; CHECK-FP16-SD-NEXT: shl v5.4h, v5.4h, #8
-; CHECK-FP16-SD-NEXT: shl v6.4h, v6.4h, #8
-; CHECK-FP16-SD-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: shl v7.4h, v7.4h, #8
-; CHECK-FP16-SD-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v4.4s, v4.4h, #0
-; CHECK-FP16-SD-NEXT: sshr v5.4h, v5.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v6.4h, v6.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v7.4h, v7.4h, #8
-; CHECK-FP16-SD-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-FP16-SD-NEXT: sshll v16.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.4s, v4.4s
-; CHECK-FP16-SD-NEXT: sshll v5.4s, v5.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v6.4s, v6.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v17.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: sshll v6.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: sshll v7.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v16.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: sshll2 v17.4s, v3.8h, #0
; CHECK-FP16-SD-NEXT: sshll v18.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-FP16-SD-NEXT: sshll v19.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: scvtf v1.4s, v16.4s
+; CHECK-FP16-SD-NEXT: scvtf v1.4s, v4.4s
+; CHECK-FP16-SD-NEXT: scvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT: scvtf v3.4s, v5.4s
; CHECK-FP16-SD-NEXT: scvtf v2.4s, v6.4s
-; CHECK-FP16-SD-NEXT: scvtf v3.4s, v17.4s
-; CHECK-FP16-SD-NEXT: scvtf v4.4s, v5.4s
-; CHECK-FP16-SD-NEXT: scvtf v5.4s, v18.4s
-; CHECK-FP16-SD-NEXT: scvtf v6.4s, v7.4s
-; CHECK-FP16-SD-NEXT: scvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT: scvtf v4.4s, v7.4s
+; CHECK-FP16-SD-NEXT: scvtf v7.4s, v16.4s
+; CHECK-FP16-SD-NEXT: scvtf v5.4s, v17.4s
+; CHECK-FP16-SD-NEXT: scvtf v6.4s, v18.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f32:
@@ -5164,78 +5092,82 @@ entry:
define <32 x float> @utofp_v32i8_v32f32(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f32:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: bic v4.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-NOFP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d5, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT: bic v4.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v5.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v6.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v2.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-NOFP16-SD-NEXT: bic v7.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: bic v3.4h, #255, lsl #8
-; CHECK-NOFP16-SD-NEXT: ushll v16.4s, v0.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.4s, v4.4h, #0
; CHECK-NOFP16-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v2.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v1.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-NOFP16-SD-NEXT: ushll v19.4s, v3.4h, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v16.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v5.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v6.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v17.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v18.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v7.4s
-; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v6.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.4s, v19.4s
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f32:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: zip1 v4.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-FP16-SD-NEXT: bic v4.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: zip1 v5.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v6.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip1 v7.8b, v3.8b, v0.8b
-; CHECK-FP16-SD-NEXT: zip2 v3.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d5, v0.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v1.d[1]
+; CHECK-FP16-SD-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT: bic v4.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v5.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v6.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v2.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v1.4h, #255, lsl #8
; CHECK-FP16-SD-NEXT: bic v7.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: bic v3.4h, #255, lsl #8
-; CHECK-FP16-SD-NEXT: ushll v16.4s, v0.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v4.4s
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v17.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v16.4s, v4.4h, #0
; CHECK-FP16-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v17.4s, v2.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v18.4s, v1.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-FP16-SD-NEXT: ushll v19.4s, v3.4h, #0
-; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v16.4s
-; CHECK-FP16-SD-NEXT: ucvtf v4.4s, v5.4s
-; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v6.4s
-; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v17.4s
-; CHECK-FP16-SD-NEXT: ucvtf v5.4s, v18.4s
-; CHECK-FP16-SD-NEXT: ucvtf v6.4s, v7.4s
-; CHECK-FP16-SD-NEXT: ucvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT: ushll v18.4s, v6.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT: ucvtf v4.4s, v1.4s
+; CHECK-FP16-SD-NEXT: ucvtf v6.4s, v17.4s
+; CHECK-FP16-SD-NEXT: ucvtf v3.4s, v16.4s
+; CHECK-FP16-SD-NEXT: ucvtf v1.4s, v5.4s
+; CHECK-FP16-SD-NEXT: ucvtf v7.4s, v18.4s
+; CHECK-FP16-SD-NEXT: ucvtf v5.4s, v19.4s
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f32:
diff --git a/llvm/test/CodeGen/AArch64/mul.ll b/llvm/test/CodeGen/AArch64/mul.ll
index dfd6f9801e54b..0ef13136c1152 100644
--- a/llvm/test/CodeGen/AArch64/mul.ll
+++ b/llvm/test/CodeGen/AArch64/mul.ll
@@ -96,10 +96,8 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: mul v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: xtn v1.8b, v0.8h
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
; CHECK-SD-NEXT: stur b0, [x0, #2]
diff --git a/llvm/test/CodeGen/AArch64/sub.ll b/llvm/test/CodeGen/AArch64/sub.ll
index 4516cf821af20..34d7b57c8bf45 100644
--- a/llvm/test/CodeGen/AArch64/sub.ll
+++ b/llvm/test/CodeGen/AArch64/sub.ll
@@ -94,9 +94,7 @@ define void @v3i8(ptr %p1, ptr %p2) {
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
; CHECK-SD-NEXT: ldr s1, [x1]
-; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: sub v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: usubl v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index 2a082b34193d8..98cf59011b51f 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -147,15 +147,9 @@ define <2 x float> @extract_v2f32_nxv16f32_2(<vscale x 16 x float> %arg) {
define <4 x i1> @extract_v4i1_nxv32i1_0(<vscale x 32 x i1> %arg) {
; CHECK-LABEL: extract_v4i1_nxv32i1_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
-; CHECK-NEXT: umov w8, v1.b[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: umov w9, v1.b[2]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: umov w8, v1.b[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%ext = call <4 x i1> @llvm.vector.extract.v4i1.nxv32i1(<vscale x 32 x i1> %arg, i64 0)
ret <4 x i1> %ext
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
index 0be662720c416..dc8eef3b056d1 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
@@ -238,8 +238,7 @@ define <2 x i1> @extract_v2i1_nxv2i1(<vscale x 2 x i1> %inmask) {
; CHECK-LABEL: extract_v2i1_nxv2i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1
-; CHECK-NEXT: mov v0.s[1], v0.s[2]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: ret
%mask = call <2 x i1> @llvm.vector.extract.v2i1.nxv2i1(<vscale x 2 x i1> %inmask, i64 0)
ret <2 x i1> %mask
@@ -248,15 +247,8 @@ define <2 x i1> @extract_v2i1_nxv2i1(<vscale x 2 x i1> %inmask) {
define <4 x i1> @extract_v4i1_nxv4i1(<vscale x 4 x i1> %inmask) {
; CHECK-LABEL: extract_v4i1_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: mov w9, v1.s[2]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: mov w8, v1.s[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
%mask = call <4 x i1> @llvm.vector.extract.v4i1.nxv4i1(<vscale x 4 x i1> %inmask, i64 0)
ret <4 x i1> %mask
@@ -265,23 +257,8 @@ define <4 x i1> @extract_v4i1_nxv4i1(<vscale x 4 x i1> %inmask) {
define <8 x i1> @extract_v8i1_nxv8i1(<vscale x 8 x i1> %inmask) {
; CHECK-LABEL: extract_v8i1_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.h, p0/z, #1 // =0x1
-; CHECK-NEXT: umov w8, v1.h[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: umov w9, v1.h[2]
-; CHECK-NEXT: mov v0.b[1], w8
-; CHECK-NEXT: umov w8, v1.h[3]
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: umov w9, v1.h[4]
-; CHECK-NEXT: mov v0.b[3], w8
-; CHECK-NEXT: umov w8, v1.h[5]
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: umov w9, v1.h[6]
-; CHECK-NEXT: mov v0.b[5], w8
-; CHECK-NEXT: umov w8, v1.h[7]
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: mov v0.b[7], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: ret
%mask = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> %inmask, i64 0)
ret <8 x i1> %mask
@@ -293,22 +270,6 @@ define <16 x i1> @extract_v16i1_nxv16i1(<vscale x 16 x i1> %inmask) {
; CHECK-LABEL: extract_v16i1_nxv16i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
-; CHECK-NEXT: mov v1.16b, v0.16b
-; CHECK-NEXT: mov v0.b[1], v0.b[1]
-; CHECK-NEXT: mov v0.b[2], v1.b[2]
-; CHECK-NEXT: mov v0.b[3], v1.b[3]
-; CHECK-NEXT: mov v0.b[4], v1.b[4]
-; CHECK-NEXT: mov v0.b[5], v1.b[5]
-; CHECK-NEXT: mov v0.b[6], v1.b[6]
-; CHECK-NEXT: mov v0.b[7], v1.b[7]
-; CHECK-NEXT: mov v0.b[8], v1.b[8]
-; CHECK-NEXT: mov v0.b[9], v1.b[9]
-; CHECK-NEXT: mov v0.b[10], v1.b[10]
-; CHECK-NEXT: mov v0.b[11], v1.b[11]
-; CHECK-NEXT: mov v0.b[12], v1.b[12]
-; CHECK-NEXT: mov v0.b[13], v1.b[13]
-; CHECK-NEXT: mov v0.b[14], v1.b[14]
-; CHECK-NEXT: mov v0.b[15], v1.b[15]
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%mask = call <16 x i1> @llvm.vector.extract.v16i1.nxv16i1(<vscale x 16 x i1> %inmask, i64 0)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
index eb938224b5375..03a974ad3d1a3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
@@ -58,20 +58,18 @@ define void @masked_gather_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
define void @masked_gather_v8i8(ptr %a, ptr %b) #0 {
; VBITS_GE_256-LABEL: masked_gather_v8i8:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldr d0, [x0]
+; VBITS_GE_256-NEXT: ldp s0, s1, [x0]
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0
-; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #8
-; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: sshr v0.4h, v0.4h, #8
-; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0
+; VBITS_GE_256-NEXT: ushll v1.8h, v1.8b, #0
+; VBITS_GE_256-NEXT: mov v0.d[1], v1.d[0]
+; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: mov d1, v0.d[1]
; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s
+; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z1.d]
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
index 07239f3c52204..96c5857a4ad67 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
@@ -58,25 +58,20 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 {
; VBITS_GE_256-NEXT: ldr d0, [x0]
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
-; VBITS_GE_256-NEXT: cmeq v1.8b, v0.8b, #0
-; VBITS_GE_256-NEXT: zip1 v3.8b, v0.8b, v0.8b
+; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0
; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT: zip1 v2.8b, v1.8b, v0.8b
-; VBITS_GE_256-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; VBITS_GE_256-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
-; VBITS_GE_256-NEXT: shl v2.4h, v2.4h, #8
-; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: cmeq v1.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h
+; VBITS_GE_256-NEXT: mov d0, v0.d[1]
+; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h
+; VBITS_GE_256-NEXT: mov d1, v1.d[1]
; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
-; VBITS_GE_256-NEXT: sshr v2.4h, v2.4h, #8
-; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8
-; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
-; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s
-; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z2.d]
; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
index 41e4a38fad90b..a4df344968b7a 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-reshuffle.ll
@@ -8,15 +8,8 @@ target triple = "aarch64-unknown-linux-gnu"
define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) #0 {
; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: mov w9, v1.s[2]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: mov w8, v1.s[3]
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
%el0 = extractelement <vscale x 4 x i1> %a, i32 0
%el1 = extractelement <vscale x 4 x i1> %a, i32 1
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
index df449f79cc9b5..00536fbfa28d1 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
@@ -168,30 +168,29 @@ define void @test_compressstore_v2i64(ptr %p, <2 x i64> %vec, <2 x i1> %mask) {
define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {
; CHECK-BASE-LABEL: test_compressstore_v8i32:
; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-BASE-NEXT: zip1 v3.8b, v2.8b, v0.8b
+; CHECK-BASE-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-BASE-NEXT: adrp x8, .LCPI11_0
-; CHECK-BASE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-BASE-NEXT: ldr d5, [x8, :lo12:.LCPI11_0]
; CHECK-BASE-NEXT: ptrue p0.s
-; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-BASE-NEXT: ldr d4, [x8, :lo12:.LCPI11_0]
; CHECK-BASE-NEXT: ptrue p1.s, vl4
-; CHECK-BASE-NEXT: shl v4.4h, v3.4h, #15
+; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: shl v3.4h, v2.4h, #15
+; CHECK-BASE-NEXT: cmlt v3.4h, v3.4h, #0
+; CHECK-BASE-NEXT: and v3.8b, v3.8b, v4.8b
+; CHECK-BASE-NEXT: ushll2 v4.4s, v2.8h, #0
; CHECK-BASE-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-BASE-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-BASE-NEXT: cmlt v4.4h, v4.4h, #0
+; CHECK-BASE-NEXT: addv h3, v3.4h
; CHECK-BASE-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-BASE-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-BASE-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-BASE-NEXT: cmpne p2.s, p1/z, z3.s, #0
-; CHECK-BASE-NEXT: addv h4, v4.4h
-; CHECK-BASE-NEXT: cntp x9, p2, p2.s
-; CHECK-BASE-NEXT: compact z0.s, p2, z0.s
-; CHECK-BASE-NEXT: fmov w8, s4
+; CHECK-BASE-NEXT: fmov w8, s3
+; CHECK-BASE-NEXT: shl v3.4s, v4.4s, #31
+; CHECK-BASE-NEXT: cmpne p2.s, p1/z, z2.s, #0
; CHECK-BASE-NEXT: and w8, w8, #0xf
; CHECK-BASE-NEXT: fmov s4, w8
+; CHECK-BASE-NEXT: cntp x9, p2, p2.s
+; CHECK-BASE-NEXT: compact z0.s, p2, z0.s
; CHECK-BASE-NEXT: cnt z4.s, p0/m, z4.s
-; CHECK-BASE-NEXT: cmpne p0.s, p1/z, z2.s, #0
+; CHECK-BASE-NEXT: cmpne p0.s, p1/z, z3.s, #0
; CHECK-BASE-NEXT: whilelo p1.s, xzr, x9
; CHECK-BASE-NEXT: cntp x8, p0, p0.s
; CHECK-BASE-NEXT: compact z1.s, p0, z1.s
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
index bcbf10d848cde..7b0d12e4111e3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
@@ -10,13 +10,8 @@ target triple = "aarch64-unknown-linux-gnu"
define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) {
; CHECK-LABEL: extract_subvector_v8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z0.b, z0.b[4]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: extract_subvector_v8i1:
@@ -43,13 +38,8 @@ define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) {
define <4 x i8> @extract_subvector_v8i8(<8 x i8> %op) {
; CHECK-LABEL: extract_subvector_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z0.b, z0.b[4]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: extract_subvector_v8i8:
@@ -117,9 +107,11 @@ define <2 x i16> @extract_subvector_v4i16(<4 x i16> %op) {
;
; NONEON-NOSVE-LABEL: extract_subvector_v4i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index 8a14f65b3d1f7..c0c9bb5eaf2a8 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -86,22 +86,22 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) {
;
; NONEON-NOSVE-LABEL: sext_v4i3_v4i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-64]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
-; NONEON-NOSVE-NEXT: stp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp w9, w8, [sp, #24]
-; NONEON-NOSVE-NEXT: ldp w11, w10, [sp, #16]
+; NONEON-NOSVE-NEXT: sub sp, sp, #48
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 48
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrh w11, [sp, #12]
; NONEON-NOSVE-NEXT: sbfx x8, x8, #0, #3
; NONEON-NOSVE-NEXT: sbfx x9, x9, #0, #3
; NONEON-NOSVE-NEXT: sbfx x10, x10, #0, #3
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #48]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT: sbfx x8, x11, #0, #3
-; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #16]
; NONEON-NOSVE-NEXT: stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: add sp, sp, #48
; NONEON-NOSVE-NEXT: ret
%b = sext <4 x i3> %a to <4 x i64>
store <4 x i64> %b, ptr %out
@@ -683,20 +683,18 @@ define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
;
; NONEON-NOSVE-LABEL: sext_v4i8_v4i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT: str q0, [sp, #-64]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
-; NONEON-NOSVE-NEXT: stp d1, d0, [sp, #16]
-; NONEON-NOSVE-NEXT: ldrsb x8, [sp, #28]
-; NONEON-NOSVE-NEXT: ldrsb x9, [sp, #24]
-; NONEON-NOSVE-NEXT: ldrsb x10, [sp, #16]
-; NONEON-NOSVE-NEXT: ldrsb x11, [sp, #20]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #48]
-; NONEON-NOSVE-NEXT: stp x10, x11, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: sub sp, sp, #48
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 48
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrsb x8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrsb x9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldrsb x10, [sp, #12]
+; NONEON-NOSVE-NEXT: ldrsb x11, [sp, #14]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: stp x10, x11, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #16]
; NONEON-NOSVE-NEXT: stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: add sp, sp, #48
; NONEON-NOSVE-NEXT: ret
%b = sext <4 x i8> %a to <4 x i64>
store <4 x i64>%b, ptr %out
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
index 583dde21d054d..f9e9bb6c09597 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
@@ -2314,21 +2314,11 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; CHECK-LABEL: masked_load_v8f32:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov z1.b, z0.b[3]
-; CHECK-NEXT: mov z2.b, z0.b[2]
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: mov z3.b, z0.b[1]
-; CHECK-NEXT: mov z4.b, z0.b[7]
-; CHECK-NEXT: mov z5.b, z0.b[6]
-; CHECK-NEXT: mov z6.b, z0.b[5]
-; CHECK-NEXT: mov z7.b, z0.b[4]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z2.h, z5.h, z4.h
-; CHECK-NEXT: zip1 z3.h, z7.h, z6.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: zip1 z1.s, z3.s, z2.s
+; CHECK-NEXT: mov x8, #4 // =0x4
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #8
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: lsl z0.s, z0.s, #31
@@ -2338,8 +2328,8 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
; CHECK-NEXT: cmpne p2.s, p0/z, z1.s, #0
; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]
-; CHECK-NEXT: ld1w { z1.s }, p2/z, [x0, x8, lsl #2]
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ld1w { z1.s }, p2/z, [x0, x8, lsl #2]
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z1
; CHECK-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
index 0fa6fca1065c6..19dc5ff64b360 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
@@ -1011,30 +1011,20 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) {
; CHECK-LABEL: masked_store_v8f32:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov z1.b, z0.b[7]
-; CHECK-NEXT: mov z2.b, z0.b[6]
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: mov z3.b, z0.b[5]
-; CHECK-NEXT: mov z4.b, z0.b[4]
-; CHECK-NEXT: mov z5.b, z0.b[3]
-; CHECK-NEXT: mov z6.b, z0.b[2]
-; CHECK-NEXT: mov z7.b, z0.b[1]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z2.h, z4.h, z3.h
-; CHECK-NEXT: zip1 z3.h, z6.h, z5.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z7.h
-; CHECK-NEXT: zip1 z1.s, z2.s, z1.s
-; CHECK-NEXT: zip1 z0.s, z0.s, z3.s
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: mov x8, #4 // =0x4
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #8
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: lsl z1.s, z1.s, #31
+; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: asr z1.s, z1.s, #31
+; CHECK-NEXT: lsl z1.s, z1.s, #31
; CHECK-NEXT: asr z0.s, z0.s, #31
+; CHECK-NEXT: asr z1.s, z1.s, #31
+; CHECK-NEXT: cmpne p2.s, p0/z, z0.s, #0
; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
; CHECK-NEXT: mov z1.s, #0 // =0x0
-; CHECK-NEXT: cmpne p2.s, p0/z, z0.s, #0
; CHECK-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
; CHECK-NEXT: st1w { z1.s }, p2, [x0]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
index 720e0fc77a8c3..572b166649807 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reshuffle.ll
@@ -11,12 +11,7 @@ define <4 x i1> @reshuffle_v4i1_nxv4i1(<vscale x 4 x i1> %a) {
; CHECK-LABEL: reshuffle_v4i1_nxv4i1:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
-; CHECK-NEXT: mov z1.s, z0.s[3]
-; CHECK-NEXT: mov z2.s, z0.s[2]
-; CHECK-NEXT: mov z3.s, z0.s[1]
-; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
-; CHECK-NEXT: zip1 z0.h, z0.h, z3.h
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: ret
%el0 = extractelement <vscale x 4 x i1> %a, i32 0
%el1 = extractelement <vscale x 4 x i1> %a, i32 1
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index df78c2394a942..373edc9361131 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -173,15 +173,12 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v2.8h, v0.8h, v1.8h
; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h
; CHECK-NEXT: str q2, [x0]
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: shl v1.4s, v1.4s, #31
-; CHECK-NEXT: shl v3.4s, v0.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index 0a3d94d8ba952..fa8a343258fa7 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -204,15 +204,12 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
; CHECK-NEXT: str q4, [x0]
; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h
-; CHECK-NEXT: xtn v2.8b, v2.8h
-; CHECK-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v3.4s, v3.4s, #31
; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: cmlt v0.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-fcvt.ll b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
index e8a9a830dced4..cc83df8adad0c 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
@@ -18,16 +18,11 @@ define <4 x float> @sitofp_v4i8_float(<4 x i8> %a) {
define <8 x float> @sitofp_v8i8_float(<8 x i8> %a) {
; CHECK-LABEL: sitofp_v8i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NEXT: sshr v1.4h, v1.4h, #8
-; CHECK-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: sshll v2.4s, v0.4h, #0
-; CHECK-NEXT: scvtf v0.4s, v1.4s
-; CHECK-NEXT: scvtf v1.4s, v2.4s
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NEXT: scvtf v0.4s, v0.4s
; CHECK-NEXT: ret
%1 = sitofp <8 x i8> %a to <8 x float>
ret <8 x float> %1
@@ -36,27 +31,16 @@ define <8 x float> @sitofp_v8i8_float(<8 x i8> %a) {
define <16 x float> @sitofp_v16i8_float(<16 x i8> %a) {
; CHECK-LABEL: sitofp_v16i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: shl v2.4h, v2.4h, #8
-; CHECK-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NEXT: sshr v2.4h, v2.4h, #8
-; CHECK-NEXT: shl v3.4h, v3.4h, #8
-; CHECK-NEXT: shl v1.4h, v1.4h, #8
-; CHECK-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-NEXT: sshr v3.4h, v3.4h, #8
-; CHECK-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-NEXT: sshll v4.4s, v0.4h, #0
; CHECK-NEXT: scvtf v0.4s, v2.4s
-; CHECK-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-NEXT: sshll v5.4s, v1.4h, #0
-; CHECK-NEXT: scvtf v1.4s, v4.4s
-; CHECK-NEXT: scvtf v2.4s, v3.4s
-; CHECK-NEXT: scvtf v3.4s, v5.4s
+; CHECK-NEXT: scvtf v3.4s, v3.4s
+; CHECK-NEXT: scvtf v1.4s, v1.4s
+; CHECK-NEXT: scvtf v2.4s, v4.4s
; CHECK-NEXT: ret
%1 = sitofp <16 x i8> %a to <16 x float>
ret <16 x float> %1
@@ -129,14 +113,15 @@ define <4 x float> @uitofp_v4i8_float(<4 x i8> %a) {
define <8 x float> @uitofp_v8i8_float(<8 x i8> %a) {
; CHECK-LABEL: uitofp_v8i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v1.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NEXT: ushll v2.4s, v0.4h, #0
-; CHECK-NEXT: ucvtf v0.4s, v1.4s
-; CHECK-NEXT: ucvtf v1.4s, v2.4s
+; CHECK-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ucvtf v1.4s, v1.4s
; CHECK-NEXT: ret
%1 = uitofp <8 x i8> %a to <8 x float>
ret <8 x float> %1
@@ -145,23 +130,24 @@ define <8 x float> @uitofp_v8i8_float(<8 x i8> %a) {
define <16 x float> @uitofp_v16i8_float(<16 x i8> %a) {
; CHECK-LABEL: uitofp_v16i8_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-NEXT: bic v0.4h, #255, lsl #8
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NEXT: bic v3.4h, #255, lsl #8
; CHECK-NEXT: bic v1.4h, #255, lsl #8
-; CHECK-NEXT: ushll v4.4s, v0.4h, #0
-; CHECK-NEXT: ucvtf v0.4s, v2.4s
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v5.4s, v1.4h, #0
-; CHECK-NEXT: ucvtf v1.4s, v4.4s
-; CHECK-NEXT: ucvtf v2.4s, v3.4s
-; CHECK-NEXT: ucvtf v3.4s, v5.4s
+; CHECK-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll v4.4s, v2.4h, #0
+; CHECK-NEXT: ushll v5.4s, v3.4h, #0
+; CHECK-NEXT: ucvtf v0.4s, v0.4s
+; CHECK-NEXT: ucvtf v2.4s, v1.4s
+; CHECK-NEXT: ucvtf v3.4s, v4.4s
+; CHECK-NEXT: ucvtf v1.4s, v5.4s
; CHECK-NEXT: ret
%1 = uitofp <16 x i8> %a to <16 x float>
ret <16 x float> %1
>From 83916919f58e5288e73f1a84e94d492ae2da2d99 Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Mon, 3 Aug 2026 21:45:12 -0700
Subject: [PATCH 2/5] Handle more types in performBuildVectorCombine and fold
anyext(build_vector())
Change-Id: I747b2267db61442629e13442c40c37ce25c8504a
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 +
.../Target/AArch64/AArch64ISelLowering.cpp | 72 +-
.../aarch64-neon-vector-insert-uaddlv.ll | 6 +-
.../AArch64/aarch64_be-shuffle-vector.ll | 4 +-
llvm/test/CodeGen/AArch64/bitcast-extend.ll | 32 +-
.../CodeGen/AArch64/combine-storetomstore.ll | 356 +++---
llvm/test/CodeGen/AArch64/itofp.ll | 1044 ++++++++---------
llvm/test/CodeGen/AArch64/neon-bitcast.ll | 15 +-
llvm/test/CodeGen/AArch64/neon-extadd.ll | 140 +--
llvm/test/CodeGen/AArch64/sext.ll | 64 +-
.../AArch64/vec3-loads-ext-trunc-stores.ll | 168 +--
llvm/test/CodeGen/AArch64/vec_uaddo.ll | 31 +-
llvm/test/CodeGen/AArch64/vec_umulo.ll | 39 +-
llvm/test/CodeGen/AArch64/vector-compress.ll | 7 +-
llvm/test/CodeGen/AArch64/vector-fcvt.ll | 204 ++--
llvm/test/CodeGen/AArch64/vselect-ext.ll | 18 +-
llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 167 +--
llvm/test/CodeGen/AArch64/zext.ll | 60 +-
llvm/test/CodeGen/X86/avx512-mask-op.ll | 34 +-
19 files changed, 1161 insertions(+), 1309 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 595cc5d653de6..3f4442a7fcde8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -16785,6 +16785,15 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
return SCC;
}
+ // Build vector can be truncating. All operands have the same type. Check if
+ // we can do the build vector in a wider type.
+ // ty1 aext(ty2 build_vector()) -> ty1 build_vector() of wider type.
+ if (N0.getOpcode() == ISD::BUILD_VECTOR &&
+ VT.getScalarSizeInBits() <=
+ N0.getOperand(0).getValueType().getScalarSizeInBits() &&
+ (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)))
+ return DAG.getBuildVector(VT, DL, N0->ops());
+
if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
return NewCtPop;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b81ef10a71bc2..3b0179739e0ce 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23801,27 +23801,73 @@ static SDValue performBuildVectorCombine(SDNode *N,
}
}
- // For now, only consider the v2i32 and v4i16 cases, which arises as a result
- // of legalization.
- if (VT != MVT::v2i32 && VT != MVT::v4i16)
+ if (DCI.isBeforeLegalize() || !VT.isInteger() ||
+ !PreExtractVT.isFixedLengthVector())
return SDValue();
// Not sure if we will ever hit the non-extend case.
if (PreExtractVT.getScalarSizeInBits() >= VT.getScalarSizeInBits())
return SDValue();
- EVT ExtVT = PreExtractVT.changeVectorElementType(*DAG.getContext(),
- VT.getVectorElementType());
+ auto TryExtractHalf = [&TLI, &DAG, &DCI, &DL](SDValue Vector, unsigned Idx,
+ bool &IsExtractHi) {
+ EVT ExtractVT =
+ Vector.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
- if (!TLI.isTypeLegal(ExtVT) && !DCI.isBeforeLegalize())
- return SDValue();
+ if (!TLI.isTypeLegal(ExtractVT))
+ return SDValue();
+ if (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(ISD::EXTRACT_SUBVECTOR, ExtractVT))
+ return SDValue();
- if (DCI.isBeforeLegalizeOps() ||
- (TLI.isOperationLegal(ISD::EXTRACT_SUBVECTOR, VT) &&
- TLI.isOperationLegal(ISD::ANY_EXTEND, ExtVT)))
- return DAG.getExtractSubvector(
- DL, VT, DAG.getAnyExtOrTrunc(VecToExtract, DL, ExtVT), ExtractIdx);
- return SDValue();
+ unsigned ExtractNumElem = ExtractVT.getVectorMinNumElements();
+ IsExtractHi = Idx >= ExtractNumElem;
+ unsigned ExtractIdx = IsExtractHi ? ExtractNumElem : 0;
+ return DAG.getExtractSubvector(DL, ExtractVT, Vector, ExtractIdx);
+ };
+
+ auto TryAnyExtendWiden = [&TLI, &DAG, &DCI, &DL](SDValue Vector) {
+ EVT ExtendVT =
+ Vector.getValueType().widenIntegerVectorElementType(*DAG.getContext());
+
+ if (!TLI.isTypeLegal(ExtendVT))
+ return SDValue();
+ if (!DCI.isBeforeLegalizeOps() &&
+ !TLI.isOperationLegal(ISD::ANY_EXTEND, ExtendVT))
+ return SDValue();
+ return DAG.getAnyExtOrTrunc(Vector, DL, ExtendVT);
+ };
+
+ SDValue CurrentVec = VecToExtract;
+ EVT CurrentTy = PreExtractVT;
+
+ if (CurrentTy.is128BitVector()) {
+ // We can't extend it. Extract first.
+ bool IsExtractHi = false;
+ if (!(CurrentVec = TryExtractHalf(CurrentVec, ExtractIdx, IsExtractHi)))
+ return SDValue();
+ CurrentTy = CurrentVec.getValueType();
+ if (IsExtractHi)
+ ExtractIdx -= CurrentTy.getVectorMinNumElements();
+ }
+
+ while (CurrentTy.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
+ // Unpack (extend and extract) until we reach the desired element type.
+ if (!(CurrentVec = TryAnyExtendWiden(CurrentVec)))
+ return SDValue();
+
+ CurrentTy = CurrentVec.getValueType();
+ if (CurrentTy.getVectorMinNumElements() == VT.getVectorMinNumElements())
+ continue;
+
+ bool IsExtractHi = false;
+ if (!(CurrentVec = TryExtractHalf(CurrentVec, ExtractIdx, IsExtractHi)))
+ return SDValue();
+ CurrentTy = CurrentVec.getValueType();
+ if (IsExtractHi)
+ ExtractIdx -= CurrentTy.getVectorMinNumElements();
+ }
+ return CurrentVec;
}
// A special combine for the sqdmulh family of instructions.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index bbb9e98a39e31..91eda8d552397 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -281,8 +281,7 @@ define void @insert_vec_v16i8_uaddlv_from_v8i8(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.8b h1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.b v2[0], v1[0]
-; CHECK-NEXT: zip1.8b v2, v2, v2
+; CHECK-NEXT: mov.h v2[0], v1[0]
; CHECK-NEXT: bic.4h v2, #255, lsl #8
; CHECK-NEXT: ushll.4s v2, v2, #0
; CHECK-NEXT: ucvtf.4s v2, v2
@@ -458,8 +457,7 @@ define void @insert_vec_v16i8_uaddlv_from_v4i32(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4s d0, v0
; CHECK-NEXT: stp q2, q2, [x0, #32]
-; CHECK-NEXT: mov.b v1[0], v0[0]
-; CHECK-NEXT: zip1.8b v1, v1, v1
+; CHECK-NEXT: mov.h v1[0], v0[0]
; CHECK-NEXT: bic.4h v1, #255, lsl #8
; CHECK-NEXT: ushll.4s v1, v1, #0
; CHECK-NEXT: ucvtf.4s v1, v1
diff --git a/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll b/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
index f88d88cf1661c..06e215ab703ec 100644
--- a/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64_be-shuffle-vector.ll
@@ -10,7 +10,7 @@ define <4 x i16> @test_reconstructshuffle(<16 x i8> %a, <16 x i8> %b) nounwind {
; CHECKLE-NEXT: mov v2.b[2], v0.b[2]
; CHECKLE-NEXT: mov v2.b[4], v0.b[1]
; CHECKLE-NEXT: mov v2.b[6], v0.b[0]
-; CHECKLE-NEXT: zip2 v0.8b, v1.8b, v0.8b
+; CHECKLE-NEXT: zip2 v0.8b, v1.8b, v1.8b
; CHECKLE-NEXT: add v0.4h, v2.4h, v0.4h
; CHECKLE-NEXT: bic v0.4h, #255, lsl #8
; CHECKLE-NEXT: ret
@@ -26,7 +26,7 @@ define <4 x i16> @test_reconstructshuffle(<16 x i8> %a, <16 x i8> %b) nounwind {
; CHECKBE-NEXT: mov v2.b[2], v0.b[2]
; CHECKBE-NEXT: mov v2.b[4], v0.b[1]
; CHECKBE-NEXT: mov v2.b[6], v0.b[0]
-; CHECKBE-NEXT: zip2 v0.8b, v1.8b, v0.8b
+; CHECKBE-NEXT: zip2 v0.8b, v1.8b, v1.8b
; CHECKBE-NEXT: add v0.4h, v2.4h, v0.4h
; CHECKBE-NEXT: bic v0.4h, #255, lsl #8
; CHECKBE-NEXT: rev64 v0.4h, v0.4h
diff --git a/llvm/test/CodeGen/AArch64/bitcast-extend.ll b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
index 60ef112c979f3..853640ffa70d6 100644
--- a/llvm/test/CodeGen/AArch64/bitcast-extend.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
@@ -61,16 +61,14 @@ define <4 x i32> @z_i32_v4i32(i32 %x) {
define <4 x i64> @z_i32_v4i64(i32 %x) {
; CHECK-SD-LABEL: z_i32_v4i64:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: movi v1.2d, #0x000000000000ff
-; CHECK-SD-NEXT: mov b2, v0.b[0]
-; CHECK-SD-NEXT: mov b3, v0.b[2]
-; CHECK-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-SD-NEXT: ushll v0.2d, v2.2s, #0
-; CHECK-SD-NEXT: ushll v2.2d, v3.2s, #0
-; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT: fmov s1, w0
+; CHECK-SD-NEXT: movi v0.2d, #0x000000000000ff
+; CHECK-SD-NEXT: zip1 v1.8b, v1.8b, v1.8b
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: ushll2 v2.2d, v1.4s, #0
+; CHECK-SD-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-SD-NEXT: and v1.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: and v0.16b, v3.16b, v0.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: z_i32_v4i64:
@@ -150,16 +148,14 @@ define <4 x i64> @s_i32_v4i64(i32 %x) {
; CHECK-SD-LABEL: s_i32_v4i64:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov b1, v0.b[0]
-; CHECK-SD-NEXT: mov b2, v0.b[2]
-; CHECK-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-SD-NEXT: ushll v0.2d, v1.2s, #0
-; CHECK-SD-NEXT: ushll v1.2d, v2.2s, #0
-; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: ushll2 v1.2d, v0.4s, #0
+; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56
-; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #56
+; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56
; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #56
+; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #56
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: s_i32_v4i64:
diff --git a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
index 3b0573b7e5c28..bdfd58787437e 100644
--- a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
@@ -183,37 +183,31 @@ define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %m
define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -265,37 +259,31 @@ define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1>
define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v8f64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: mov b7, v4.b[2]
-; SVE-NEXT: mov b16, v4.b[0]
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: mov x9, #6 // =0x6
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: mov x8, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[3]
-; SVE-NEXT: mov v16.b[4], v4.b[1]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]
+; SVE-NEXT: st1d { z3.d }, p1, [x0, x8, lsl #3]
; SVE-NEXT: mov x8, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]
; SVE-NEXT: st1d { z0.d }, p4, [x0]
; SVE-NEXT: ret
@@ -345,35 +333,32 @@ define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1>
define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: mov x9, #8 // =0x8
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpne p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpne p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpne p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpne p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpne p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpne p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
; SVE-NEXT: st1w { z2.s }, p2, [x0, x9, lsl #2]
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load
@@ -470,64 +455,60 @@ define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1>
; SVE-NEXT: fmov s4, w9
; SVE-NEXT: ldr w9, [sp, #8]
; SVE-NEXT: fmov s5, w11
-; SVE-NEXT: mov v7.b[1], w2
+; SVE-NEXT: mov v7.h[1], w2
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
; SVE-NEXT: fmov s6, w9
; SVE-NEXT: ldr w9, [sp, #152]
-; SVE-NEXT: mov v4.b[1], w8
+; SVE-NEXT: mov v4.h[1], w8
; SVE-NEXT: ldr w8, [sp, #16]
-; SVE-NEXT: mov v5.b[1], w10
-; SVE-NEXT: mov v6.b[1], w8
+; SVE-NEXT: mov v5.h[1], w10
+; SVE-NEXT: mov v6.h[1], w8
; SVE-NEXT: ldr w8, [sp, #88]
-; SVE-NEXT: mov v7.b[2], w3
-; SVE-NEXT: mov v4.b[2], w8
+; SVE-NEXT: mov v7.h[2], w3
+; SVE-NEXT: mov v4.h[2], w8
; SVE-NEXT: ldr w8, [sp, #24]
-; SVE-NEXT: mov v5.b[2], w9
+; SVE-NEXT: mov v5.h[2], w9
; SVE-NEXT: ldr w9, [sp, #160]
-; SVE-NEXT: mov v6.b[2], w8
+; SVE-NEXT: mov v6.h[2], w8
; SVE-NEXT: ldr w8, [sp, #96]
-; SVE-NEXT: mov v7.b[3], w4
-; SVE-NEXT: mov v4.b[3], w8
+; SVE-NEXT: mov v7.h[3], w4
+; SVE-NEXT: mov v4.h[3], w8
; SVE-NEXT: ldr w8, [sp, #32]
-; SVE-NEXT: mov v5.b[3], w9
+; SVE-NEXT: mov v5.h[3], w9
; SVE-NEXT: ldr w9, [sp, #168]
-; SVE-NEXT: mov v6.b[3], w8
+; SVE-NEXT: mov v6.h[3], w8
; SVE-NEXT: ldr w8, [sp, #104]
-; SVE-NEXT: mov v7.b[4], w5
-; SVE-NEXT: mov v4.b[4], w8
+; SVE-NEXT: mov v7.h[4], w5
+; SVE-NEXT: mov v4.h[4], w8
; SVE-NEXT: ldr w8, [sp, #40]
-; SVE-NEXT: mov v5.b[4], w9
+; SVE-NEXT: mov v5.h[4], w9
; SVE-NEXT: ldr w9, [sp, #176]
-; SVE-NEXT: mov v6.b[4], w8
+; SVE-NEXT: mov v6.h[4], w8
; SVE-NEXT: ldr w8, [sp, #112]
-; SVE-NEXT: mov v7.b[5], w6
-; SVE-NEXT: mov v4.b[5], w8
+; SVE-NEXT: mov v7.h[5], w6
+; SVE-NEXT: mov v4.h[5], w8
; SVE-NEXT: ldr w8, [sp, #48]
-; SVE-NEXT: mov v5.b[5], w9
+; SVE-NEXT: mov v5.h[5], w9
; SVE-NEXT: ldr w9, [sp, #184]
-; SVE-NEXT: mov v6.b[5], w8
+; SVE-NEXT: mov v6.h[5], w8
; SVE-NEXT: ldr w8, [sp, #120]
-; SVE-NEXT: mov v7.b[6], w7
-; SVE-NEXT: mov v4.b[6], w8
+; SVE-NEXT: mov v7.h[6], w7
+; SVE-NEXT: mov v4.h[6], w8
; SVE-NEXT: ldr w8, [sp, #56]
-; SVE-NEXT: mov v5.b[6], w9
+; SVE-NEXT: mov v5.h[6], w9
; SVE-NEXT: ldr w9, [sp, #192]
-; SVE-NEXT: mov v6.b[6], w8
+; SVE-NEXT: mov v6.h[6], w8
; SVE-NEXT: ldr w8, [sp, #128]
-; SVE-NEXT: mov v4.b[7], w8
+; SVE-NEXT: mov v4.h[7], w8
; SVE-NEXT: ldr w8, [sp, #64]
-; SVE-NEXT: mov v5.b[7], w9
+; SVE-NEXT: mov v5.h[7], w9
; SVE-NEXT: ldr w9, [sp]
-; SVE-NEXT: mov v6.b[7], w8
+; SVE-NEXT: mov v6.h[7], w8
; SVE-NEXT: mov x8, #16 // =0x10
-; SVE-NEXT: mov v7.b[7], w9
-; SVE-NEXT: ushll v4.8h, v4.8b, #0
-; SVE-NEXT: ushll v5.8h, v5.8b, #0
-; SVE-NEXT: ushll v6.8h, v6.8b, #0
-; SVE-NEXT: ushll v7.8h, v7.8b, #0
+; SVE-NEXT: mov v7.h[7], w9
; SVE-NEXT: shl v4.8h, v4.8h, #15
; SVE-NEXT: shl v5.8h, v5.8h, #15
; SVE-NEXT: shl v6.8h, v6.8h, #15
@@ -748,35 +729,32 @@ define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr,
define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; SVE-LABEL: test_masked_store_success_invert_mask_v16i32:
; SVE: // %bb.0:
-; SVE-NEXT: mov d5, v4.d[1]
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b
-; SVE-NEXT: mov x8, #4 // =0x4
-; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b
+; SVE-NEXT: ushll2 v5.8h, v4.16b, #0
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: mov x8, #12 // =0xc
; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
-; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b
-; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b
-; SVE-NEXT: ushll v6.4s, v6.4h, #0
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: ushll2 v6.4s, v5.8h, #0
+; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: ushll2 v7.4s, v4.8h, #0
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: shl v6.4s, v6.4s, #31
-; SVE-NEXT: ushll v7.4s, v7.4h, #0
-; SVE-NEXT: ushll v5.4s, v5.4h, #0
+; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: shl v7.4s, v7.4s, #31
; SVE-NEXT: shl v4.4s, v4.4s, #31
; SVE-NEXT: cmpge p1.s, p0/z, z6.s, #0
-; SVE-NEXT: shl v7.4s, v7.4s, #31
-; SVE-NEXT: shl v5.4s, v5.4s, #31
+; SVE-NEXT: cmpge p2.s, p0/z, z5.s, #0
+; SVE-NEXT: cmpge p3.s, p0/z, z7.s, #0
; SVE-NEXT: cmpge p4.s, p0/z, z4.s, #0
-; SVE-NEXT: cmpge p2.s, p0/z, z7.s, #0
-; SVE-NEXT: cmpge p3.s, p0/z, z5.s, #0
-; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
; SVE-NEXT: mov x8, #8 // =0x8
-; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: st1w { z2.s }, p2, [x0, x8, lsl #2]
-; SVE-NEXT: mov x8, #12 // =0xc
-; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: mov x8, #4 // =0x4
+; SVE-NEXT: st1w { z1.s }, p3, [x0, x8, lsl #2]
+; SVE-NEXT: st1w { z0.s }, p4, [x0]
; SVE-NEXT: ret
%load = load <16 x i32>, ptr %ptr, align 32
%sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x
@@ -938,68 +916,58 @@ define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %p
; SVE: // %bb.0:
; SVE-NEXT: ldp d16, d17, [sp]
; SVE-NEXT: ptrue p1.d, vl2
-; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov x9, #6 // =0x6
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-NEXT: mov x8, #6 // =0x6
+; SVE-NEXT: mov x8, #2 // =0x2
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov b18, v16.b[0]
-; SVE-NEXT: mov b19, v16.b[4]
-; SVE-NEXT: mov b20, v17.b[4]
-; SVE-NEXT: mov b21, v16.b[6]
-; SVE-NEXT: mov b22, v16.b[2]
-; SVE-NEXT: mov b23, v17.b[6]
-; SVE-NEXT: mov b24, v17.b[2]
-; SVE-NEXT: mov b25, v17.b[0]
-; SVE-NEXT: mov v18.b[4], v16.b[1]
-; SVE-NEXT: mov v19.b[4], v16.b[5]
-; SVE-NEXT: mov v20.b[4], v17.b[5]
-; SVE-NEXT: mov v21.b[4], v16.b[7]
-; SVE-NEXT: mov v22.b[4], v16.b[3]
-; SVE-NEXT: mov v23.b[4], v17.b[7]
-; SVE-NEXT: mov v24.b[4], v17.b[3]
-; SVE-NEXT: mov v25.b[4], v17.b[1]
+; SVE-NEXT: ushll v16.8h, v16.8b, #0
+; SVE-NEXT: ushll v17.8h, v17.8b, #0
+; SVE-NEXT: ushll v18.4s, v16.4h, #0
+; SVE-NEXT: ushll2 v16.4s, v16.8h, #0
+; SVE-NEXT: ushll2 v19.4s, v17.8h, #0
+; SVE-NEXT: ushll v17.4s, v17.4h, #0
+; SVE-NEXT: ushll2 v21.2d, v16.4s, #0
+; SVE-NEXT: ushll2 v20.2d, v18.4s, #0
+; SVE-NEXT: ushll v16.2d, v16.2s, #0
+; SVE-NEXT: ushll2 v22.2d, v19.4s, #0
+; SVE-NEXT: ushll v19.2d, v19.2s, #0
+; SVE-NEXT: ushll2 v23.2d, v17.4s, #0
+; SVE-NEXT: ushll v17.2d, v17.2s, #0
; SVE-NEXT: ushll v18.2d, v18.2s, #0
-; SVE-NEXT: ushll v17.2d, v19.2s, #0
-; SVE-NEXT: ushll v19.2d, v21.2s, #0
-; SVE-NEXT: ushll v21.2d, v25.2s, #0
-; SVE-NEXT: shl v16.2d, v18.2d, #63
-; SVE-NEXT: ushll v18.2d, v20.2s, #0
-; SVE-NEXT: ushll v20.2d, v23.2s, #0
-; SVE-NEXT: shl v17.2d, v17.2d, #63
-; SVE-NEXT: shl v19.2d, v19.2d, #63
; SVE-NEXT: shl v21.2d, v21.2d, #63
-; SVE-NEXT: cmpne p0.d, p1/z, z16.d, #0
-; SVE-NEXT: ushll v16.2d, v22.2s, #0
-; SVE-NEXT: shl v18.2d, v18.2d, #63
-; SVE-NEXT: ushll v22.2d, v24.2s, #0
; SVE-NEXT: shl v20.2d, v20.2d, #63
-; SVE-NEXT: cmpne p2.d, p1/z, z17.d, #0
-; SVE-NEXT: cmpne p3.d, p1/z, z19.d, #0
; SVE-NEXT: shl v16.2d, v16.2d, #63
-; SVE-NEXT: cmlt v17.2d, v18.2d, #0
-; SVE-NEXT: shl v18.2d, v22.2d, #63
-; SVE-NEXT: ldp q19, q22, [x1, #32]
+; SVE-NEXT: shl v19.2d, v19.2d, #63
+; SVE-NEXT: shl v23.2d, v23.2d, #63
+; SVE-NEXT: shl v22.2d, v22.2d, #63
+; SVE-NEXT: shl v17.2d, v17.2d, #63
+; SVE-NEXT: shl v18.2d, v18.2d, #63
+; SVE-NEXT: cmpne p2.d, p1/z, z21.d, #0
+; SVE-NEXT: cmpne p0.d, p1/z, z20.d, #0
; SVE-NEXT: cmpne p4.d, p1/z, z16.d, #0
-; SVE-NEXT: cmlt v16.2d, v20.2d, #0
-; SVE-NEXT: cmlt v20.2d, v21.2d, #0
-; SVE-NEXT: cmlt v18.2d, v18.2d, #0
-; SVE-NEXT: bif v6.16b, v19.16b, v17.16b
-; SVE-NEXT: ldp q17, q19, [x1]
-; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]
-; SVE-NEXT: mov x9, #2 // =0x2
-; SVE-NEXT: st1d { z3.d }, p3, [x0, x8, lsl #3]
-; SVE-NEXT: mov v2.16b, v16.16b
-; SVE-NEXT: mov v3.16b, v20.16b
-; SVE-NEXT: st1d { z1.d }, p4, [x0, x9, lsl #3]
-; SVE-NEXT: mov v1.16b, v18.16b
-; SVE-NEXT: st1d { z0.d }, p0, [x0]
-; SVE-NEXT: bsl v2.16b, v7.16b, v22.16b
-; SVE-NEXT: bsl v3.16b, v4.16b, v17.16b
-; SVE-NEXT: bsl v1.16b, v5.16b, v19.16b
-; SVE-NEXT: stp q6, q2, [x1, #32]
-; SVE-NEXT: stp q3, q1, [x1]
+; SVE-NEXT: cmlt v16.2d, v19.2d, #0
+; SVE-NEXT: cmlt v23.2d, v23.2d, #0
+; SVE-NEXT: cmlt v22.2d, v22.2d, #0
+; SVE-NEXT: cmlt v17.2d, v17.2d, #0
+; SVE-NEXT: ldp q21, q20, [x1, #32]
+; SVE-NEXT: cmpne p3.d, p1/z, z18.d, #0
+; SVE-NEXT: ldp q19, q18, [x1]
+; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]
+; SVE-NEXT: mov x9, #4 // =0x4
+; SVE-NEXT: mov v3.16b, v16.16b
+; SVE-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]
+; SVE-NEXT: mov v1.16b, v23.16b
+; SVE-NEXT: bif v7.16b, v20.16b, v22.16b
+; SVE-NEXT: st1d { z2.d }, p4, [x0, x9, lsl #3]
+; SVE-NEXT: mov v2.16b, v17.16b
+; SVE-NEXT: bsl v3.16b, v6.16b, v21.16b
+; SVE-NEXT: st1d { z0.d }, p3, [x0]
+; SVE-NEXT: bsl v1.16b, v5.16b, v18.16b
+; SVE-NEXT: bsl v2.16b, v4.16b, v19.16b
+; SVE-NEXT: stp q3, q7, [x1, #32]
+; SVE-NEXT: stp q2, q1, [x1]
; SVE-NEXT: ret
%load = load <8 x i64>, ptr %ptr1, align 32
%load2 = load <8 x i64>, ptr %ptr2, align 32
@@ -1084,40 +1052,34 @@ define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i
define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {
; SVE-LABEL: test_masked_store_unaligned_v8i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $d4 killed $d4 def $q4
-; SVE-NEXT: mov b5, v4.b[4]
-; SVE-NEXT: mov b6, v4.b[6]
-; SVE-NEXT: add x8, x0, #33
-; SVE-NEXT: mov b7, v4.b[0]
-; SVE-NEXT: mov b16, v4.b[2]
-; SVE-NEXT: add x9, x0, #49
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: add x8, x0, #49
+; SVE-NEXT: add x9, x0, #33
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: mov v5.b[4], v4.b[5]
-; SVE-NEXT: mov v6.b[4], v4.b[7]
-; SVE-NEXT: mov v7.b[4], v4.b[1]
-; SVE-NEXT: mov v16.b[4], v4.b[3]
-; SVE-NEXT: ushll v4.2d, v5.2s, #0
-; SVE-NEXT: ushll v5.2d, v6.2s, #0
-; SVE-NEXT: ushll v6.2d, v7.2s, #0
-; SVE-NEXT: ushll v7.2d, v16.2s, #0
-; SVE-NEXT: shl v4.2d, v4.2d, #63
-; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: ushll2 v5.4s, v4.8h, #0
+; SVE-NEXT: ushll v4.4s, v4.4h, #0
+; SVE-NEXT: ushll2 v6.2d, v5.4s, #0
+; SVE-NEXT: ushll v5.2d, v5.2s, #0
+; SVE-NEXT: ushll2 v7.2d, v4.4s, #0
+; SVE-NEXT: ushll v4.2d, v4.2s, #0
; SVE-NEXT: shl v6.2d, v6.2d, #63
-; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0
-; SVE-NEXT: shl v4.2d, v7.2d, #63
+; SVE-NEXT: shl v5.2d, v5.2d, #63
+; SVE-NEXT: shl v7.2d, v7.2d, #63
+; SVE-NEXT: shl v4.2d, v4.2d, #63
+; SVE-NEXT: cmpne p1.d, p0/z, z6.d, #0
; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0
-; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0
+; SVE-NEXT: cmpne p3.d, p0/z, z7.d, #0
; SVE-NEXT: cmpne p4.d, p0/z, z4.d, #0
-; SVE-NEXT: st1d { z2.d }, p1, [x8]
-; SVE-NEXT: add x8, x0, #1
-; SVE-NEXT: st1d { z3.d }, p2, [x9]
-; SVE-NEXT: add x9, x0, #17
-; SVE-NEXT: st1d { z0.d }, p3, [x8]
-; SVE-NEXT: st1d { z1.d }, p4, [x9]
+; SVE-NEXT: st1d { z3.d }, p1, [x8]
+; SVE-NEXT: add x8, x0, #17
+; SVE-NEXT: st1d { z2.d }, p2, [x9]
+; SVE-NEXT: add x9, x0, #1
+; SVE-NEXT: st1d { z1.d }, p3, [x8]
+; SVE-NEXT: st1d { z0.d }, p4, [x9]
; SVE-NEXT: ret
%ptr_i8 = getelementptr i8, ptr %ptr, i64 1
%ptr_vec = bitcast ptr %ptr_i8 to ptr
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index e966614eb7238..147377a6434e2 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -2355,60 +2355,52 @@ entry:
define <8 x double> @stofp_v8i8_v8f64(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-SD-NEXT: mov b1, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-SD-NEXT: mov b1, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d2, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
-; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v5.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f64:
@@ -2448,54 +2440,46 @@ entry:
define <8 x double> @utofp_v8i8_v8f64(<8 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[6]
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NOFP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d3, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d4, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[6]
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-FP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d3, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d4, v0.d[1]
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v1.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f64:
@@ -2535,108 +2519,94 @@ entry:
define <16 x double> @stofp_v16i8_v16f64(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b6, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b7, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b16, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d5, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v7.2s, v7.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v7.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v20.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d1, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b2, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b6, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b7, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b16, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-FP16-SD-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d4, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d5, v1.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v0.d[1]
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: shl v7.2s, v7.2s, #24
; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-FP16-SD-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: scvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v16.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v17.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v5.2d
+; CHECK-FP16-SD-NEXT: scvtf v7.2d, v19.2d
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v20.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f64:
@@ -2698,94 +2668,80 @@ entry:
define <16 x double> @utofp_v16i8_v16f64(<16 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b4, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b5, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b6, v0.b[6]
+; CHECK-NOFP16-SD-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-NOFP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov b7, v2.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b16, v2.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b17, v2.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b18, v2.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d5, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d6, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v19.2d
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b4, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b5, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov b6, v0.b[6]
+; CHECK-FP16-SD-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-FP16-SD-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov b7, v2.b[0]
-; CHECK-FP16-SD-NEXT: mov b16, v2.b[2]
-; CHECK-FP16-SD-NEXT: mov b17, v2.b[4]
-; CHECK-FP16-SD-NEXT: mov b18, v2.b[6]
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-FP16-SD-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: mov d5, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d6, v2.d[1]
+; CHECK-FP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v0.d[1]
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-FP16-SD-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v19.2d
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f64:
@@ -2847,222 +2803,194 @@ entry:
define <32 x double> @stofp_v32i8_v32f64(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov d3, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b5, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b20, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b21, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b18, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b22, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: mov d16, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b2, v3.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b4, v3.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b6, v3.b[4]
-; CHECK-NOFP16-SD-NEXT: mov b7, v3.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v5.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v20.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v22.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: mov b23, v16.b[0]
-; CHECK-NOFP16-SD-NEXT: mov v2.b[4], v3.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v3.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v3.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v3.b[7]
-; CHECK-NOFP16-SD-NEXT: mov b3, v0.b[2]
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v7.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d6, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d18, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: shl v5.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v17.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v20.2s, v20.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b24, v16.b[4]
-; CHECK-NOFP16-SD-NEXT: mov v23.b[4], v16.b[1]
-; CHECK-NOFP16-SD-NEXT: shl v18.2s, v18.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d23, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d21, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d22, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v19.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d7, v7.d[1]
+; CHECK-NOFP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v16.2s, v16.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v5.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v17.2s, v17.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: shl v0.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v6.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b7, v16.b[2]
+; CHECK-NOFP16-SD-NEXT: shl v18.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v23.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v20.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: mov d17, v17.d[1]
+; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v16.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v20.2s, v20.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov b21, v16.b[6]
-; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v18.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v21.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-NOFP16-SD-NEXT: shl v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: mov v24.b[4], v16.b[5]
-; CHECK-NOFP16-SD-NEXT: sshr v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v16.b[3]
-; CHECK-NOFP16-SD-NEXT: sshll v20.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v16.b[7]
+; CHECK-NOFP16-SD-NEXT: sshll v18.2d, v18.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshr v23.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
; CHECK-NOFP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v22.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v19.2d, v19.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v20.2d, v20.2d
-; CHECK-NOFP16-SD-NEXT: shl v22.2s, v24.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v6.2s, v6.2s, #24
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v18.2d, v18.2d
; CHECK-NOFP16-SD-NEXT: shl v7.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v4.2s, #24
-; CHECK-NOFP16-SD-NEXT: stp q17, q5, [x8, #160]
-; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v18.2s, #24
-; CHECK-NOFP16-SD-NEXT: shl v17.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT: shl v17.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NOFP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: shl v21.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: scvtf v19.2d, v19.2d
-; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v2.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v1.2s, v1.2s, #24
-; CHECK-NOFP16-SD-NEXT: stp q0, q20, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-NOFP16-SD-NEXT: sshr v5.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v16.2s, v20.2s, #24
+; CHECK-NOFP16-SD-NEXT: stp q2, q18, [x8, #192]
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q4, q6, [x8, #224]
+; CHECK-NOFP16-SD-NEXT: sshr v4.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v23.2s, #0
; CHECK-NOFP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshr v0.2s, v7.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v2.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v7.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT: sshr v17.2s, v19.2s, #24
; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: sshr v18.2s, v21.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshr v20.2s, v22.2s, #24
-; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-NOFP16-SD-NEXT: sshll v4.2d, v4.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q16, q19, [x8, #32]
-; CHECK-NOFP16-SD-NEXT: sshll v17.2d, v18.2s, #0
-; CHECK-NOFP16-SD-NEXT: sshll v16.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-NOFP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: stp q5, q3, [x8]
-; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v5.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-NOFP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: stp q1, q6, [x8, #160]
+; CHECK-NOFP16-SD-NEXT: sshll v1.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: sshll v6.2d, v17.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-NOFP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: stp q4, q5, [x8, #96]
-; CHECK-NOFP16-SD-NEXT: stp q3, q0, [x8, #192]
-; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v1.2d
-; CHECK-NOFP16-SD-NEXT: stp q16, q17, [x8, #224]
-; CHECK-NOFP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: stp q3, q4, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: scvtf v3.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: scvtf v0.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: stp q3, q2, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q0, q1, [x8]
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov d3, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov b5, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov b20, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b21, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b18, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b22, v0.b[4]
-; CHECK-FP16-SD-NEXT: mov d16, v1.d[1]
-; CHECK-FP16-SD-NEXT: mov b2, v3.b[0]
-; CHECK-FP16-SD-NEXT: mov b4, v3.b[2]
-; CHECK-FP16-SD-NEXT: mov b6, v3.b[4]
-; CHECK-FP16-SD-NEXT: mov b7, v3.b[6]
-; CHECK-FP16-SD-NEXT: mov v5.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov v20.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v22.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: mov b23, v16.b[0]
-; CHECK-FP16-SD-NEXT: mov v2.b[4], v3.b[1]
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v3.b[3]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v3.b[5]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v3.b[7]
-; CHECK-FP16-SD-NEXT: mov b3, v0.b[2]
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v17.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v7.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: mov d6, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-FP16-SD-NEXT: mov d18, v2.d[1]
+; CHECK-FP16-SD-NEXT: shl v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: shl v5.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v17.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: shl v20.2s, v20.2s, #24
-; CHECK-FP16-SD-NEXT: mov b24, v16.b[4]
-; CHECK-FP16-SD-NEXT: mov v23.b[4], v16.b[1]
-; CHECK-FP16-SD-NEXT: shl v18.2s, v18.2s, #24
-; CHECK-FP16-SD-NEXT: shl v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT: mov d23, v1.d[1]
+; CHECK-FP16-SD-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: mov d21, v0.d[1]
+; CHECK-FP16-SD-NEXT: mov d22, v3.d[1]
+; CHECK-FP16-SD-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v19.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT: mov d7, v7.d[1]
+; CHECK-FP16-SD-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: shl v16.2s, v16.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-FP16-SD-NEXT: sshr v5.2s, v5.2s, #24
-; CHECK-FP16-SD-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v17.2s, v17.2s, #24
-; CHECK-FP16-SD-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: shl v0.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: shl v6.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: mov b7, v16.b[2]
+; CHECK-FP16-SD-NEXT: shl v18.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT: shl v23.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: shl v20.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: mov d17, v17.d[1]
+; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v16.2s, v16.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v20.2s, v20.2s, #24
-; CHECK-FP16-SD-NEXT: mov b21, v16.b[6]
-; CHECK-FP16-SD-NEXT: sshll v17.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-FP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v18.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT: shl v21.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-FP16-SD-NEXT: shl v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: mov v24.b[4], v16.b[5]
-; CHECK-FP16-SD-NEXT: sshr v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT: shl v22.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v16.b[3]
-; CHECK-FP16-SD-NEXT: sshll v20.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v16.b[7]
+; CHECK-FP16-SD-NEXT: sshll v18.2d, v18.2s, #0
+; CHECK-FP16-SD-NEXT: sshr v23.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
; CHECK-FP16-SD-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v16.2s, v22.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v19.2d, v19.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v20.2d, v20.2d
-; CHECK-FP16-SD-NEXT: shl v22.2s, v24.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v6.2s, v6.2s, #24
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT: scvtf v18.2d, v18.2d
; CHECK-FP16-SD-NEXT: shl v7.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v4.2s, v4.2s, #24
-; CHECK-FP16-SD-NEXT: stp q17, q5, [x8, #160]
-; CHECK-FP16-SD-NEXT: sshr v5.2s, v18.2s, #24
-; CHECK-FP16-SD-NEXT: shl v17.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT: shl v17.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-FP16-SD-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: shl v21.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: scvtf v19.2d, v19.2d
-; CHECK-FP16-SD-NEXT: sshr v2.2s, v2.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v1.2s, v1.2s, #24
-; CHECK-FP16-SD-NEXT: stp q0, q20, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-FP16-SD-NEXT: sshr v5.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v16.2s, v20.2s, #24
+; CHECK-FP16-SD-NEXT: stp q2, q18, [x8, #192]
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: stp q4, q6, [x8, #224]
+; CHECK-FP16-SD-NEXT: sshr v4.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v23.2s, #0
; CHECK-FP16-SD-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-FP16-SD-NEXT: sshr v0.2s, v7.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v7.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v2.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v7.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT: sshr v17.2s, v19.2s, #24
; CHECK-FP16-SD-NEXT: scvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: sshr v18.2s, v21.2s, #24
-; CHECK-FP16-SD-NEXT: sshr v20.2s, v22.2s, #24
-; CHECK-FP16-SD-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-FP16-SD-NEXT: sshll v4.2d, v4.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT: sshll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT: scvtf v5.2d, v5.2d
; CHECK-FP16-SD-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-FP16-SD-NEXT: stp q16, q19, [x8, #32]
-; CHECK-FP16-SD-NEXT: sshll v17.2d, v18.2s, #0
-; CHECK-FP16-SD-NEXT: sshll v16.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v0.2d
; CHECK-FP16-SD-NEXT: scvtf v4.2d, v4.2d
-; CHECK-FP16-SD-NEXT: stp q5, q3, [x8]
-; CHECK-FP16-SD-NEXT: scvtf v3.2d, v7.2d
-; CHECK-FP16-SD-NEXT: scvtf v5.2d, v6.2d
-; CHECK-FP16-SD-NEXT: scvtf v17.2d, v17.2d
-; CHECK-FP16-SD-NEXT: scvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT: stp q1, q6, [x8, #160]
+; CHECK-FP16-SD-NEXT: sshll v1.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: sshll v6.2d, v17.2s, #0
+; CHECK-FP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-FP16-SD-NEXT: scvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: stp q4, q5, [x8, #96]
-; CHECK-FP16-SD-NEXT: stp q3, q0, [x8, #192]
-; CHECK-FP16-SD-NEXT: scvtf v0.2d, v1.2d
-; CHECK-FP16-SD-NEXT: stp q16, q17, [x8, #224]
-; CHECK-FP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT: stp q3, q4, [x8, #64]
+; CHECK-FP16-SD-NEXT: scvtf v3.2d, v16.2d
+; CHECK-FP16-SD-NEXT: scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: scvtf v0.2d, v6.2d
+; CHECK-FP16-SD-NEXT: stp q3, q2, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q0, q1, [x8]
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f64:
@@ -3184,192 +3112,164 @@ entry:
define <32 x double> @utofp_v32i8_v32f64(<32 x i8> %a) {
; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f64:
; CHECK-NOFP16-SD: // %bb.0: // %entry
-; CHECK-NOFP16-SD-NEXT: mov b6, v1.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b7, v1.b[4]
-; CHECK-NOFP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-NOFP16-SD-NEXT: mov b16, v1.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b17, v1.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-NOFP16-SD-NEXT: mov b20, v0.b[4]
-; CHECK-NOFP16-SD-NEXT: movi d5, #0x0000ff000000ff
-; CHECK-NOFP16-SD-NEXT: mov b24, v0.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b25, v0.b[0]
-; CHECK-NOFP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-NOFP16-SD-NEXT: mov v6.b[4], v1.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v1.b[5]
-; CHECK-NOFP16-SD-NEXT: mov b18, v3.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b21, v3.b[2]
-; CHECK-NOFP16-SD-NEXT: mov b23, v3.b[4]
-; CHECK-NOFP16-SD-NEXT: mov v16.b[4], v1.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v17.b[4], v1.b[1]
-; CHECK-NOFP16-SD-NEXT: mov b1, v3.b[6]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-NOFP16-SD-NEXT: mov v20.b[4], v0.b[5]
-; CHECK-NOFP16-SD-NEXT: mov v24.b[4], v0.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v25.b[4], v0.b[1]
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v6.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v18.b[4], v3.b[1]
-; CHECK-NOFP16-SD-NEXT: mov v21.b[4], v3.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v23.b[4], v3.b[5]
-; CHECK-NOFP16-SD-NEXT: and v16.8b, v16.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v17.8b, v17.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v1.b[4], v3.b[7]
-; CHECK-NOFP16-SD-NEXT: and v3.8b, v19.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT: movi d6, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v19.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v18.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d17, v2.d[1]
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v5.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v2.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d20, v19.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d21, v3.d[1]
+; CHECK-NOFP16-SD-NEXT: mov d23, v0.d[1]
+; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT: mov d22, v18.d[1]
+; CHECK-NOFP16-SD-NEXT: and v3.8b, v3.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v16.8b, v16.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v17.8b, v17.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v0.8b, v0.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: and v20.8b, v20.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v3.2s, #0
; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v20.8b, v20.8b, v5.8b
; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b4, v2.b[0]
-; CHECK-NOFP16-SD-NEXT: mov b22, v2.b[2]
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v3.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b19, v2.b[4]
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: ushll v20.2d, v20.2s, #0
-; CHECK-NOFP16-SD-NEXT: mov b7, v2.b[6]
+; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-NOFP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: and v24.8b, v24.8b, v5.8b
; CHECK-NOFP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: mov v4.b[4], v2.b[1]
-; CHECK-NOFP16-SD-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-NOFP16-SD-NEXT: mov v22.b[4], v2.b[3]
-; CHECK-NOFP16-SD-NEXT: mov v19.b[4], v2.b[5]
-; CHECK-NOFP16-SD-NEXT: stp q3, q6, [x8, #160]
-; CHECK-NOFP16-SD-NEXT: and v6.8b, v25.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v1.8b, v1.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: mov v7.b[4], v2.b[7]
-; CHECK-NOFP16-SD-NEXT: and v2.8b, v23.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v24.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q17, q16, [x8, #128]
-; CHECK-NOFP16-SD-NEXT: and v16.8b, v21.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: stp q20, q0, [x8, #32]
-; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v0.8b, v18.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v20.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v5.8b, v23.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: stp q2, q17, [x8, #192]
+; CHECK-NOFP16-SD-NEXT: stp q4, q7, [x8, #224]
+; CHECK-NOFP16-SD-NEXT: and v7.8b, v19.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: mov d4, v1.d[1]
+; CHECK-NOFP16-SD-NEXT: and v19.8b, v21.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v1.8b, v1.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT: ushll v17.2d, v19.2s, #0
+; CHECK-NOFP16-SD-NEXT: and v2.8b, v4.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v4.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT: and v16.8b, v22.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: and v6.8b, v18.8b, v6.8b
+; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-NOFP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-NOFP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: and v17.8b, v19.8b, v5.8b
-; CHECK-NOFP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-NOFP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NOFP16-SD-NEXT: and v18.8b, v22.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT: stp q7, q4, [x8, #160]
+; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v16.2s, #0
; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-NOFP16-SD-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-NOFP16-SD-NEXT: ushll v5.2d, v7.2s, #0
-; CHECK-NOFP16-SD-NEXT: ushll v7.2d, v17.2s, #0
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NOFP16-SD-NEXT: stp q6, q3, [x8]
-; CHECK-NOFP16-SD-NEXT: ushll v3.2d, v18.2s, #0
-; CHECK-NOFP16-SD-NEXT: stp q2, q1, [x8, #224]
-; CHECK-NOFP16-SD-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v1.2d, v7.2d
-; CHECK-NOFP16-SD-NEXT: stp q0, q16, [x8, #192]
-; CHECK-NOFP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v4.2d
-; CHECK-NOFP16-SD-NEXT: stp q1, q5, [x8, #96]
-; CHECK-NOFP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: stp q3, q17, [x8, #64]
+; CHECK-NOFP16-SD-NEXT: ucvtf v0.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT: stp q1, q2, [x8, #128]
+; CHECK-NOFP16-SD-NEXT: stp q0, q3, [x8]
; CHECK-NOFP16-SD-NEXT: ret
;
; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f64:
; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: mov b6, v1.b[6]
-; CHECK-FP16-SD-NEXT: mov b7, v1.b[4]
-; CHECK-FP16-SD-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-SD-NEXT: mov b16, v1.b[2]
-; CHECK-FP16-SD-NEXT: mov b17, v1.b[0]
-; CHECK-FP16-SD-NEXT: mov b19, v0.b[6]
-; CHECK-FP16-SD-NEXT: mov b20, v0.b[4]
-; CHECK-FP16-SD-NEXT: movi d5, #0x0000ff000000ff
-; CHECK-FP16-SD-NEXT: mov b24, v0.b[2]
-; CHECK-FP16-SD-NEXT: mov b25, v0.b[0]
-; CHECK-FP16-SD-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-SD-NEXT: mov v6.b[4], v1.b[7]
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v1.b[5]
-; CHECK-FP16-SD-NEXT: mov b18, v3.b[0]
-; CHECK-FP16-SD-NEXT: mov b21, v3.b[2]
-; CHECK-FP16-SD-NEXT: mov b23, v3.b[4]
-; CHECK-FP16-SD-NEXT: mov v16.b[4], v1.b[3]
-; CHECK-FP16-SD-NEXT: mov v17.b[4], v1.b[1]
-; CHECK-FP16-SD-NEXT: mov b1, v3.b[6]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v0.b[7]
-; CHECK-FP16-SD-NEXT: mov v20.b[4], v0.b[5]
-; CHECK-FP16-SD-NEXT: mov v24.b[4], v0.b[3]
-; CHECK-FP16-SD-NEXT: mov v25.b[4], v0.b[1]
-; CHECK-FP16-SD-NEXT: and v6.8b, v6.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v18.b[4], v3.b[1]
-; CHECK-FP16-SD-NEXT: mov v21.b[4], v3.b[3]
-; CHECK-FP16-SD-NEXT: mov v23.b[4], v3.b[5]
-; CHECK-FP16-SD-NEXT: and v16.8b, v16.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v17.8b, v17.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v1.b[4], v3.b[7]
-; CHECK-FP16-SD-NEXT: and v3.8b, v19.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: ushll2 v2.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT: movi d6, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v19.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v18.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT: mov d7, v4.d[1]
+; CHECK-FP16-SD-NEXT: mov d16, v5.d[1]
+; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d17, v2.d[1]
+; CHECK-FP16-SD-NEXT: and v5.8b, v5.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v2.8b, v2.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d20, v19.d[1]
+; CHECK-FP16-SD-NEXT: mov d21, v3.d[1]
+; CHECK-FP16-SD-NEXT: mov d23, v0.d[1]
+; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT: mov d22, v18.d[1]
+; CHECK-FP16-SD-NEXT: and v3.8b, v3.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v16.8b, v16.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: and v17.8b, v17.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT: and v0.8b, v0.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT: and v20.8b, v20.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v3.2d, v3.2s, #0
; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: and v20.8b, v20.8b, v5.8b
; CHECK-FP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: mov b4, v2.b[0]
-; CHECK-FP16-SD-NEXT: mov b22, v2.b[2]
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
; CHECK-FP16-SD-NEXT: ushll v17.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v3.2s, #0
-; CHECK-FP16-SD-NEXT: mov b19, v2.b[4]
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v7.2d
-; CHECK-FP16-SD-NEXT: ushll v20.2d, v20.2s, #0
-; CHECK-FP16-SD-NEXT: mov b7, v2.b[6]
+; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-FP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: and v24.8b, v24.8b, v5.8b
; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: mov v4.b[4], v2.b[1]
-; CHECK-FP16-SD-NEXT: ucvtf v20.2d, v20.2d
-; CHECK-FP16-SD-NEXT: mov v22.b[4], v2.b[3]
-; CHECK-FP16-SD-NEXT: mov v19.b[4], v2.b[5]
-; CHECK-FP16-SD-NEXT: stp q3, q6, [x8, #160]
-; CHECK-FP16-SD-NEXT: and v6.8b, v25.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v1.8b, v1.8b, v5.8b
-; CHECK-FP16-SD-NEXT: mov v7.b[4], v2.b[7]
-; CHECK-FP16-SD-NEXT: and v2.8b, v23.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v24.2s, #0
-; CHECK-FP16-SD-NEXT: stp q17, q16, [x8, #128]
-; CHECK-FP16-SD-NEXT: and v16.8b, v21.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v4.8b, v4.8b, v5.8b
-; CHECK-FP16-SD-NEXT: stp q20, q0, [x8, #32]
-; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
-; CHECK-FP16-SD-NEXT: and v0.8b, v18.8b, v5.8b
+; CHECK-FP16-SD-NEXT: stp q5, q16, [x8, #96]
+; CHECK-FP16-SD-NEXT: ushll v16.2d, v20.2s, #0
+; CHECK-FP16-SD-NEXT: and v5.8b, v23.8b, v6.8b
+; CHECK-FP16-SD-NEXT: stp q2, q17, [x8, #192]
+; CHECK-FP16-SD-NEXT: stp q4, q7, [x8, #224]
+; CHECK-FP16-SD-NEXT: and v7.8b, v19.8b, v6.8b
+; CHECK-FP16-SD-NEXT: mov d4, v1.d[1]
+; CHECK-FP16-SD-NEXT: and v19.8b, v21.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT: and v1.8b, v1.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ushll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT: ushll v17.2d, v19.2s, #0
+; CHECK-FP16-SD-NEXT: and v2.8b, v4.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v4.2d, v16.2d
+; CHECK-FP16-SD-NEXT: and v16.8b, v22.8b, v6.8b
+; CHECK-FP16-SD-NEXT: and v6.8b, v18.8b, v6.8b
+; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT: ucvtf v7.2d, v7.2d
; CHECK-FP16-SD-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT: ucvtf v17.2d, v17.2d
; CHECK-FP16-SD-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-FP16-SD-NEXT: and v7.8b, v7.8b, v5.8b
-; CHECK-FP16-SD-NEXT: and v17.8b, v19.8b, v5.8b
-; CHECK-FP16-SD-NEXT: ucvtf v6.2d, v6.2d
-; CHECK-FP16-SD-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-FP16-SD-NEXT: and v18.8b, v22.8b, v5.8b
+; CHECK-FP16-SD-NEXT: ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT: stp q0, q5, [x8, #32]
; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT: stp q7, q4, [x8, #160]
+; CHECK-FP16-SD-NEXT: ushll v4.2d, v16.2s, #0
; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v2.2d
-; CHECK-FP16-SD-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v16.2d, v16.2d
-; CHECK-FP16-SD-NEXT: ushll v5.2d, v7.2s, #0
-; CHECK-FP16-SD-NEXT: ushll v7.2d, v17.2s, #0
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-FP16-SD-NEXT: stp q6, q3, [x8]
-; CHECK-FP16-SD-NEXT: ushll v3.2d, v18.2s, #0
-; CHECK-FP16-SD-NEXT: stp q2, q1, [x8, #224]
-; CHECK-FP16-SD-NEXT: ucvtf v5.2d, v5.2d
-; CHECK-FP16-SD-NEXT: ucvtf v1.2d, v7.2d
-; CHECK-FP16-SD-NEXT: stp q0, q16, [x8, #192]
-; CHECK-FP16-SD-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v4.2d
-; CHECK-FP16-SD-NEXT: stp q1, q5, [x8, #96]
-; CHECK-FP16-SD-NEXT: stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT: stp q3, q17, [x8, #64]
+; CHECK-FP16-SD-NEXT: ucvtf v0.2d, v6.2d
+; CHECK-FP16-SD-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT: stp q1, q2, [x8, #128]
+; CHECK-FP16-SD-NEXT: stp q0, q3, [x8]
; CHECK-FP16-SD-NEXT: ret
;
; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f64:
diff --git a/llvm/test/CodeGen/AArch64/neon-bitcast.ll b/llvm/test/CodeGen/AArch64/neon-bitcast.ll
index c6aa8701e1721..6b43dd23f45aa 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitcast.ll
@@ -554,19 +554,18 @@ define <4 x i8> @bitcast_i32_to_v4i8(i32 %word) {
define <2 x i8> @bitcast_i16_to_v2i8(i16 %word) {
; CHECK-LE-LABEL: bitcast_i16_to_v2i8:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: fmov s1, w0
-; CHECK-LE-NEXT: mov b0, v1.b[0]
-; CHECK-LE-NEXT: mov v0.b[4], v1.b[1]
-; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-LE-NEXT: fmov s0, w0
+; CHECK-LE-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-LE-NEXT: zip1 v0.4h, v0.4h, v0.4h
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: bitcast_i16_to_v2i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: fmov s0, w0
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: mov b1, v0.b[0]
-; CHECK-BE-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-BE-NEXT: rev64 v0.2s, v1.2s
+; CHECK-BE-NEXT: rev16 v0.8b, v0.8b
+; CHECK-BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-BE-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
; CHECK-BE-NEXT: ret
%ret = bitcast i16 %word to <2 x i8>
ret <2 x i8> %ret
diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll
index 01fdfa1ef2eb8..b48fb576ff1e0 100644
--- a/llvm/test/CodeGen/AArch64/neon-extadd.ll
+++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll
@@ -1368,83 +1368,69 @@ entry:
define <16 x i32> @i12(<16 x i12> %s0, <16 x i12> %s1) {
; CHECK-SD-LABEL: i12:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr w8, [sp, #32]
-; CHECK-SD-NEXT: ldr w9, [sp, #40]
-; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr w10, [sp, #128]
-; CHECK-SD-NEXT: fmov s3, w0
-; CHECK-SD-NEXT: ldr w11, [sp, #72]
-; CHECK-SD-NEXT: fmov s0, w8
-; CHECK-SD-NEXT: ldr w8, [sp]
-; CHECK-SD-NEXT: movi v16.4s, #15, msl #8
-; CHECK-SD-NEXT: fmov s5, w10
-; CHECK-SD-NEXT: ldr w10, [sp, #104]
-; CHECK-SD-NEXT: mov v1.h[1], w5
-; CHECK-SD-NEXT: fmov s2, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #8]
-; CHECK-SD-NEXT: mov v3.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[1], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #48]
-; CHECK-SD-NEXT: mov v2.h[1], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #160]
-; CHECK-SD-NEXT: mov v1.h[2], w6
-; CHECK-SD-NEXT: mov v3.h[2], w2
-; CHECK-SD-NEXT: fmov s4, w8
-; CHECK-SD-NEXT: mov v0.h[2], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #168]
-; CHECK-SD-NEXT: ldr w8, [sp, #64]
-; CHECK-SD-NEXT: mov v1.h[3], w7
-; CHECK-SD-NEXT: mov v4.h[1], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #96]
-; CHECK-SD-NEXT: fmov s6, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #136]
-; CHECK-SD-NEXT: mov v3.h[3], w3
-; CHECK-SD-NEXT: fmov s7, w9
-; CHECK-SD-NEXT: ldr w9, [sp, #16]
-; CHECK-SD-NEXT: mov v6.h[1], w11
-; CHECK-SD-NEXT: mov v5.h[1], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #80]
-; CHECK-SD-NEXT: mov v2.h[2], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #176]
-; CHECK-SD-NEXT: ldr w11, [sp, #144]
-; CHECK-SD-NEXT: mov v7.h[1], w10
-; CHECK-SD-NEXT: ldr w10, [sp, #112]
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: mov v4.h[2], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #56]
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: mov v6.h[2], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #24]
-; CHECK-SD-NEXT: mov v5.h[2], w11
-; CHECK-SD-NEXT: ldr w11, [sp, #120]
-; CHECK-SD-NEXT: mov v0.h[3], w9
-; CHECK-SD-NEXT: ldr w9, [sp, #152]
-; CHECK-SD-NEXT: mov v7.h[2], w10
-; CHECK-SD-NEXT: ldr w10, [sp, #88]
-; CHECK-SD-NEXT: mov v2.h[3], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #184]
-; CHECK-SD-NEXT: and v1.16b, v1.16b, v16.16b
-; CHECK-SD-NEXT: mov v6.h[3], w10
-; CHECK-SD-NEXT: mov v5.h[3], w9
-; CHECK-SD-NEXT: mov v4.h[3], w8
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: mov v7.h[3], w11
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-SD-NEXT: and v17.16b, v0.16b, v16.16b
-; CHECK-SD-NEXT: ushll v4.4s, v4.4h, #0
-; CHECK-SD-NEXT: and v2.16b, v2.16b, v16.16b
-; CHECK-SD-NEXT: and v0.16b, v3.16b, v16.16b
-; CHECK-SD-NEXT: ushll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT: and v3.16b, v6.16b, v16.16b
-; CHECK-SD-NEXT: and v5.16b, v5.16b, v16.16b
-; CHECK-SD-NEXT: and v4.16b, v4.16b, v16.16b
-; CHECK-SD-NEXT: and v6.16b, v7.16b, v16.16b
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-SD-NEXT: add v2.4s, v2.4s, v5.4s
-; CHECK-SD-NEXT: add v3.4s, v17.4s, v4.4s
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v6.4s
+; CHECK-SD-NEXT: ldr s2, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #40
+; CHECK-SD-NEXT: fmov s0, w4
+; CHECK-SD-NEXT: fmov s1, w0
+; CHECK-SD-NEXT: ldr s4, [sp]
+; CHECK-SD-NEXT: add x9, sp, #8
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #48
+; CHECK-SD-NEXT: ldr s6, [sp, #64]
+; CHECK-SD-NEXT: ld1 { v4.s }[1], [x9]
+; CHECK-SD-NEXT: mov v0.s[1], w5
+; CHECK-SD-NEXT: add x9, sp, #16
+; CHECK-SD-NEXT: mov v1.s[1], w1
+; CHECK-SD-NEXT: ldr s5, [sp, #96]
+; CHECK-SD-NEXT: add x11, sp, #72
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #56
+; CHECK-SD-NEXT: ldr s7, [sp, #160]
+; CHECK-SD-NEXT: ldr s16, [sp, #128]
+; CHECK-SD-NEXT: add x10, sp, #104
+; CHECK-SD-NEXT: ld1 { v4.s }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v6.s }[1], [x11]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v5.s }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #168
+; CHECK-SD-NEXT: ld1 { v16.s }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v7.s }[1], [x8]
+; CHECK-SD-NEXT: mov v0.s[2], w6
+; CHECK-SD-NEXT: mov v1.s[2], w2
+; CHECK-SD-NEXT: add x10, sp, #80
+; CHECK-SD-NEXT: add x9, sp, #112
+; CHECK-SD-NEXT: add x11, sp, #144
+; CHECK-SD-NEXT: ld1 { v6.s }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #176
+; CHECK-SD-NEXT: ld1 { v5.s }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v7.s }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v16.s }[2], [x11]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: add x9, sp, #88
+; CHECK-SD-NEXT: movi v3.4s, #15, msl #8
+; CHECK-SD-NEXT: mov v0.s[3], w7
+; CHECK-SD-NEXT: mov v1.s[3], w3
+; CHECK-SD-NEXT: ld1 { v4.s }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #120
+; CHECK-SD-NEXT: ld1 { v6.s }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: add x10, sp, #152
+; CHECK-SD-NEXT: ld1 { v5.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v7.s }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v16.s }[3], [x10]
+; CHECK-SD-NEXT: and v17.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: and v0.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT: and v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT: and v2.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: and v4.16b, v6.16b, v3.16b
+; CHECK-SD-NEXT: and v5.16b, v5.16b, v3.16b
+; CHECK-SD-NEXT: and v6.16b, v7.16b, v3.16b
+; CHECK-SD-NEXT: and v3.16b, v16.16b, v3.16b
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-SD-NEXT: add v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: add v3.4s, v1.4s, v6.4s
+; CHECK-SD-NEXT: add v1.4s, v17.4s, v5.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: i12:
diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll
index 278731ad0663a..f1fae4aafd26a 100644
--- a/llvm/test/CodeGen/AArch64/sext.ll
+++ b/llvm/test/CodeGen/AArch64/sext.ll
@@ -239,9 +239,8 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD-LABEL: sext_v3i8_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24
; CHECK-SD-NEXT: ret
@@ -398,9 +397,8 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD-LABEL: sext_v3i10_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
; CHECK-SD-NEXT: ret
@@ -1055,42 +1053,36 @@ entry:
define <16 x i32> @sext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: sext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr w8, [sp, #32]
-; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr w10, [sp, #40]
-; CHECK-SD-NEXT: ldr w11, [sp, #8]
-; CHECK-SD-NEXT: fmov s2, w9
-; CHECK-SD-NEXT: fmov s3, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #48]
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: ldr w9, [sp, #16]
-; CHECK-SD-NEXT: mov v1.h[1], w5
-; CHECK-SD-NEXT: mov v2.h[1], w11
-; CHECK-SD-NEXT: mov v3.h[1], w10
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: mov v1.h[2], w6
-; CHECK-SD-NEXT: mov v2.h[2], w9
-; CHECK-SD-NEXT: mov v3.h[2], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #56]
-; CHECK-SD-NEXT: ldr w9, [sp, #24]
-; CHECK-SD-NEXT: mov v0.h[3], w3
-; CHECK-SD-NEXT: mov v1.h[3], w7
-; CHECK-SD-NEXT: mov v2.h[3], w9
-; CHECK-SD-NEXT: mov v3.h[3], w8
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
-; CHECK-SD-NEXT: shl v1.4s, v1.4s, #22
+; CHECK-SD-NEXT: ldr s2, [sp]
+; CHECK-SD-NEXT: ldr s3, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #8
+; CHECK-SD-NEXT: add x9, sp, #40
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #16
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v1.s[1], w5
+; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #48
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #56
+; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v1.s[2], w6
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-SD-NEXT: mov v0.s[3], w3
+; CHECK-SD-NEXT: mov v1.s[3], w7
; CHECK-SD-NEXT: shl v2.4s, v2.4s, #22
; CHECK-SD-NEXT: shl v3.4s, v3.4s, #22
-; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
-; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #22
; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
+; CHECK-SD-NEXT: shl v1.4s, v1.4s, #22
; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #22
+; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
+; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #22
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: sext_v16i10_v16i32:
diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 85af93eeadbdf..0b6e1dc7c0bbb 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -40,12 +40,14 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32:
@@ -53,16 +55,16 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -78,12 +80,14 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_2:
@@ -91,16 +95,16 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -114,21 +118,25 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
define <4 x i32> @load_v3i8_to_4xi32_align_4(ptr %src) {
; CHECK-LABEL: load_v3i8_to_4xi32_align_4:
; CHECK: ; %bb.0:
-; CHECK-NEXT: ldr s0, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: ldr s1, [x0]
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_4:
; BE: // %bb.0:
-; BE-NEXT: ldr s0, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: ldr s1, [x0]
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: rev32 v1.16b, v1.16b
+; BE-NEXT: umov w8, v1.b[1]
+; BE-NEXT: umov w9, v1.b[2]
+; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: mov v1.s[2], w9
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: ret
@@ -143,12 +151,14 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #3]
; CHECK-NEXT: ldurh w9, [x0, #1]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_1:
@@ -156,16 +166,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #1]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #3]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -182,12 +192,14 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #5]
; CHECK-NEXT: ldurh w9, [x0, #3]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: zip1.8b v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: umov.b w8, v1[1]
+; CHECK-NEXT: umov.b w9, v1[2]
+; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: mov.s v1[2], w9
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_3:
@@ -195,16 +207,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #3]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #5]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -222,15 +234,15 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
; CHECK-NEXT: strh w8, [sp, #12]
-; CHECK-NEXT: ldr s0, [sp, #12]
+; CHECK-NEXT: ldr s1, [sp, #12]
+; CHECK-NEXT: ushll.8h v1, v1, #0
+; CHECK-NEXT: umov.h w8, v1[1]
+; CHECK-NEXT: mov.s v1[1], w8
; CHECK-NEXT: ldrsb w8, [x0, #2]
-; CHECK-NEXT: ushll.8h v0, v0, #0
-; CHECK-NEXT: mov.h v0[1], v0[1]
-; CHECK-NEXT: mov.h v0[2], w8
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: mov.s v1[2], w8
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
;
@@ -239,16 +251,16 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: movi v0.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s0, [sp, #12]
+; BE-NEXT: ldr s1, [sp, #12]
+; BE-NEXT: rev32 v1.8b, v1.8b
+; BE-NEXT: ushll v1.8h, v1.8b, #0
+; BE-NEXT: umov w8, v1.h[1]
+; BE-NEXT: mov v1.s[1], w8
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: rev32 v0.8b, v0.8b
-; BE-NEXT: ushll v0.8h, v0.8b, #0
-; BE-NEXT: mov v0.h[1], v0.h[1]
-; BE-NEXT: mov v0.h[2], w8
-; BE-NEXT: ushll v0.4s, v0.4h, #0
-; BE-NEXT: and v0.16b, v0.16b, v1.16b
+; BE-NEXT: mov v1.s[2], w8
+; BE-NEXT: and v0.16b, v1.16b, v0.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index 373edc9361131..a1766dfef9465 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -141,23 +141,20 @@ define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v4.16b, v0.16b, v1.16b
; CHECK-NEXT: cmhi v0.16b, v0.16b, v4.16b
; CHECK-NEXT: str q4, [x0]
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: zip1 v2.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip2 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT: zip1 v3.8b, v1.8b, v0.8b
-; CHECK-NEXT: zip2 v1.8b, v1.8b, v0.8b
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: shl v1.4s, v1.4s, #31
; CHECK-NEXT: shl v5.4s, v0.4s, #31
; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: shl v6.4s, v1.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -173,12 +170,8 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v2.8h, v0.8h, v1.8h
; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h
; CHECK-NEXT: str q2, [x0]
-; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: shl v1.4s, v1.4s, #31
-; CHECK-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index fa8a343258fa7..66757b6f7cf51 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -165,27 +165,24 @@ define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
-; CHECK-NEXT: mul v6.16b, v0.16b, v1.16b
+; CHECK-NEXT: mul v7.16b, v0.16b, v1.16b
; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: str q6, [x0]
+; CHECK-NEXT: str q7, [x0]
; CHECK-NEXT: cmtst v2.16b, v2.16b, v2.16b
-; CHECK-NEXT: mov d3, v2.d[1]
-; CHECK-NEXT: zip1 v4.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-NEXT: zip1 v5.8b, v3.8b, v0.8b
-; CHECK-NEXT: zip2 v3.8b, v3.8b, v0.8b
-; CHECK-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-NEXT: ushll v3.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NEXT: ushll2 v5.4s, v2.8h, #0
+; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0
; CHECK-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-NEXT: shl v4.4s, v4.4s, #31
-; CHECK-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: shl v5.4s, v5.4s, #31
+; CHECK-NEXT: shl v6.4s, v3.4s, #31
; CHECK-NEXT: shl v2.4s, v2.4s, #31
; CHECK-NEXT: cmlt v0.4s, v4.4s, #0
-; CHECK-NEXT: shl v5.4s, v5.4s, #31
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v2.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
-; CHECK-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v3.4s, v5.4s, #0
+; CHECK-NEXT: cmlt v1.4s, v6.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
%val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
@@ -200,16 +197,12 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
-; CHECK-NEXT: mul v4.8h, v0.8h, v1.8h
; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: str q4, [x0]
+; CHECK-NEXT: mul v3.8h, v0.8h, v1.8h
; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h
-; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
-; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
+; CHECK-NEXT: str q3, [x0]
+; CHECK-NEXT: sshll2 v1.4s, v2.8h, #0
+; CHECK-NEXT: sshll v0.4s, v2.4h, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 55c343164a1b8..74f38884fe9f6 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -413,10 +413,9 @@ define <3 x i32> @test_compress_narrow(<3 x i32> %vec, <3 x i1> %mask) {
; CHECK-NEXT: movi.2d v1, #0000000000000000
; CHECK-NEXT: mov x11, sp
; CHECK-NEXT: str s0, [sp]
-; CHECK-NEXT: mov.h v1[0], w0
-; CHECK-NEXT: mov.h v1[1], w1
-; CHECK-NEXT: mov.h v1[2], w2
-; CHECK-NEXT: ushll.4s v1, v1, #0
+; CHECK-NEXT: mov.s v1[0], w0
+; CHECK-NEXT: mov.s v1[1], w1
+; CHECK-NEXT: mov.s v1[2], w2
; CHECK-NEXT: shl.4s v1, v1, #31
; CHECK-NEXT: cmlt.4s v1, v1, #0
; CHECK-NEXT: mov.s w8, v1[1]
diff --git a/llvm/test/CodeGen/AArch64/vector-fcvt.ll b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
index cc83df8adad0c..5af84aeb6943c 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcvt.ll
@@ -228,31 +228,27 @@ define <4 x double> @sitofp_v4i8_double(<4 x i8> %a) {
define <8 x double> @sitofp_v8i8_double(<8 x i8> %a) {
; CHECK-LABEL: sitofp_v8i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov b1, v0.b[0]
-; CHECK-NEXT: mov b2, v0.b[2]
-; CHECK-NEXT: mov b3, v0.b[4]
-; CHECK-NEXT: mov b4, v0.b[6]
-; CHECK-NEXT: mov v1.b[4], v0.b[1]
-; CHECK-NEXT: mov v2.b[4], v0.b[3]
-; CHECK-NEXT: mov v3.b[4], v0.b[5]
-; CHECK-NEXT: mov v4.b[4], v0.b[7]
-; CHECK-NEXT: shl v0.2s, v1.2s, #24
-; CHECK-NEXT: shl v1.2s, v2.2s, #24
-; CHECK-NEXT: shl v2.2s, v3.2s, #24
-; CHECK-NEXT: shl v3.2s, v4.2s, #24
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d2, v1.d[1]
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v1.2s, v1.2s, #24
; CHECK-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NEXT: shl v3.2s, v3.2s, #24
; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: sshr v2.2s, v2.2s, #24
; CHECK-NEXT: sshr v3.2s, v3.2s, #24
-; CHECK-NEXT: sshll v0.2d, v0.2s, #0
; CHECK-NEXT: sshll v1.2d, v1.2s, #0
-; CHECK-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NEXT: sshll v3.2d, v3.2s, #0
; CHECK-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NEXT: scvtf v1.2d, v1.2d
-; CHECK-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NEXT: scvtf v3.2d, v3.2d
+; CHECK-NEXT: sshll v4.2d, v2.2s, #0
+; CHECK-NEXT: sshll v5.2d, v3.2s, #0
+; CHECK-NEXT: scvtf v2.2d, v1.2d
+; CHECK-NEXT: scvtf v3.2d, v4.2d
+; CHECK-NEXT: scvtf v1.2d, v5.2d
; CHECK-NEXT: ret
%1 = sitofp <8 x i8> %a to <8 x double>
ret <8 x double> %1
@@ -261,55 +257,48 @@ define <8 x double> @sitofp_v8i8_double(<8 x i8> %a) {
define <16 x double> @sitofp_v16i8_double(<16 x i8> %a) {
; CHECK-LABEL: sitofp_v16i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: mov b2, v0.b[0]
-; CHECK-NEXT: mov b3, v0.b[2]
-; CHECK-NEXT: mov b4, v0.b[4]
-; CHECK-NEXT: mov b5, v0.b[6]
-; CHECK-NEXT: mov b6, v1.b[0]
-; CHECK-NEXT: mov b7, v1.b[2]
-; CHECK-NEXT: mov b16, v1.b[4]
-; CHECK-NEXT: mov b17, v1.b[6]
-; CHECK-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NEXT: mov v6.b[4], v1.b[1]
-; CHECK-NEXT: mov v7.b[4], v1.b[3]
-; CHECK-NEXT: mov v16.b[4], v1.b[5]
-; CHECK-NEXT: mov v17.b[4], v1.b[7]
-; CHECK-NEXT: shl v0.2s, v2.2s, #24
-; CHECK-NEXT: shl v1.2s, v3.2s, #24
-; CHECK-NEXT: shl v2.2s, v4.2s, #24
-; CHECK-NEXT: shl v3.2s, v5.2s, #24
-; CHECK-NEXT: shl v4.2s, v6.2s, #24
-; CHECK-NEXT: shl v5.2s, v7.2s, #24
-; CHECK-NEXT: shl v6.2s, v16.2s, #24
-; CHECK-NEXT: shl v7.2s, v17.2s, #24
-; CHECK-NEXT: sshr v0.2s, v0.2s, #24
-; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d4, v2.d[1]
+; CHECK-NEXT: mov d5, v1.d[1]
+; CHECK-NEXT: mov d6, v3.d[1]
+; CHECK-NEXT: mov d7, v0.d[1]
+; CHECK-NEXT: shl v2.2s, v2.2s, #24
+; CHECK-NEXT: shl v1.2s, v1.2s, #24
+; CHECK-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NEXT: shl v3.2s, v3.2s, #24
+; CHECK-NEXT: shl v4.2s, v4.2s, #24
+; CHECK-NEXT: shl v5.2s, v5.2s, #24
+; CHECK-NEXT: shl v6.2s, v6.2s, #24
+; CHECK-NEXT: shl v7.2s, v7.2s, #24
; CHECK-NEXT: sshr v2.2s, v2.2s, #24
+; CHECK-NEXT: sshr v1.2s, v1.2s, #24
+; CHECK-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NEXT: sshr v3.2s, v3.2s, #24
; CHECK-NEXT: sshr v4.2s, v4.2s, #24
; CHECK-NEXT: sshr v5.2s, v5.2s, #24
; CHECK-NEXT: sshr v6.2s, v6.2s, #24
; CHECK-NEXT: sshr v7.2s, v7.2s, #24
-; CHECK-NEXT: sshll v0.2d, v0.2s, #0
-; CHECK-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-NEXT: sshll v2.2d, v2.2s, #0
-; CHECK-NEXT: sshll v3.2d, v3.2s, #0
-; CHECK-NEXT: sshll v4.2d, v4.2s, #0
+; CHECK-NEXT: sshll v1.2d, v1.2s, #0
+; CHECK-NEXT: sshll v16.2d, v0.2s, #0
+; CHECK-NEXT: sshll v18.2d, v3.2s, #0
+; CHECK-NEXT: sshll v17.2d, v4.2s, #0
; CHECK-NEXT: sshll v5.2d, v5.2s, #0
-; CHECK-NEXT: sshll v6.2d, v6.2s, #0
-; CHECK-NEXT: sshll v7.2d, v7.2s, #0
-; CHECK-NEXT: scvtf v0.2d, v0.2d
-; CHECK-NEXT: scvtf v1.2d, v1.2d
+; CHECK-NEXT: sshll v19.2d, v6.2s, #0
+; CHECK-NEXT: sshll v20.2d, v7.2s, #0
; CHECK-NEXT: scvtf v2.2d, v2.2d
-; CHECK-NEXT: scvtf v3.2d, v3.2d
-; CHECK-NEXT: scvtf v4.2d, v4.2d
-; CHECK-NEXT: scvtf v5.2d, v5.2d
-; CHECK-NEXT: scvtf v6.2d, v6.2d
-; CHECK-NEXT: scvtf v7.2d, v7.2d
+; CHECK-NEXT: scvtf v0.2d, v1.2d
+; CHECK-NEXT: scvtf v4.2d, v16.2d
+; CHECK-NEXT: scvtf v6.2d, v18.2d
+; CHECK-NEXT: scvtf v3.2d, v17.2d
+; CHECK-NEXT: scvtf v1.2d, v5.2d
+; CHECK-NEXT: scvtf v7.2d, v19.2d
+; CHECK-NEXT: scvtf v5.2d, v20.2d
; CHECK-NEXT: ret
%1 = sitofp <16 x i8> %a to <16 x double>
ret <16 x double> %1
@@ -381,28 +370,24 @@ define <4 x double> @uitofp_v4i8_double(<4 x i8> %a) {
define <8 x double> @uitofp_v8i8_double(<8 x i8> %a) {
; CHECK-LABEL: uitofp_v8i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov b2, v0.b[0]
-; CHECK-NEXT: mov b3, v0.b[2]
-; CHECK-NEXT: mov b4, v0.b[4]
-; CHECK-NEXT: mov b5, v0.b[6]
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NEXT: mov v2.b[4], v0.b[1]
-; CHECK-NEXT: mov v3.b[4], v0.b[3]
-; CHECK-NEXT: mov v4.b[4], v0.b[5]
-; CHECK-NEXT: mov v5.b[4], v0.b[7]
-; CHECK-NEXT: and v0.8b, v2.8b, v1.8b
-; CHECK-NEXT: and v2.8b, v3.8b, v1.8b
-; CHECK-NEXT: and v3.8b, v4.8b, v1.8b
-; CHECK-NEXT: and v1.8b, v5.8b, v1.8b
+; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d3, v2.d[1]
+; CHECK-NEXT: mov d4, v0.d[1]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: and v2.8b, v2.8b, v1.8b
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NEXT: and v1.8b, v4.8b, v1.8b
; CHECK-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NEXT: ushll v4.2d, v1.2s, #0
; CHECK-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NEXT: ucvtf v1.2d, v2.2d
-; CHECK-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NEXT: ucvtf v3.2d, v4.2d
+; CHECK-NEXT: ushll v3.2d, v3.2s, #0
+; CHECK-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-NEXT: ucvtf v1.2d, v1.2d
; CHECK-NEXT: ret
%1 = uitofp <8 x i8> %a to <8 x double>
ret <8 x double> %1
@@ -411,48 +396,41 @@ define <8 x double> @uitofp_v8i8_double(<8 x i8> %a) {
define <16 x double> @uitofp_v16i8_double(<16 x i8> %a) {
; CHECK-LABEL: uitofp_v16i8_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov b3, v0.b[0]
-; CHECK-NEXT: mov b4, v0.b[2]
-; CHECK-NEXT: mov b5, v0.b[4]
-; CHECK-NEXT: mov b6, v0.b[6]
+; CHECK-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v0.8h, v0.16b, #0
; CHECK-NEXT: movi d1, #0x0000ff000000ff
-; CHECK-NEXT: mov b7, v2.b[0]
-; CHECK-NEXT: mov b16, v2.b[2]
-; CHECK-NEXT: mov b17, v2.b[4]
-; CHECK-NEXT: mov b18, v2.b[6]
-; CHECK-NEXT: mov v3.b[4], v0.b[1]
-; CHECK-NEXT: mov v4.b[4], v0.b[3]
-; CHECK-NEXT: mov v5.b[4], v0.b[5]
-; CHECK-NEXT: mov v6.b[4], v0.b[7]
-; CHECK-NEXT: mov v7.b[4], v2.b[1]
-; CHECK-NEXT: mov v16.b[4], v2.b[3]
-; CHECK-NEXT: mov v17.b[4], v2.b[5]
-; CHECK-NEXT: mov v18.b[4], v2.b[7]
-; CHECK-NEXT: and v0.8b, v3.8b, v1.8b
-; CHECK-NEXT: and v2.8b, v4.8b, v1.8b
-; CHECK-NEXT: and v3.8b, v5.8b, v1.8b
-; CHECK-NEXT: and v4.8b, v6.8b, v1.8b
-; CHECK-NEXT: and v5.8b, v7.8b, v1.8b
-; CHECK-NEXT: and v6.8b, v16.8b, v1.8b
-; CHECK-NEXT: and v7.8b, v17.8b, v1.8b
-; CHECK-NEXT: and v1.8b, v18.8b, v1.8b
-; CHECK-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NEXT: ushll v2.2d, v2.2s, #0
+; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d5, v3.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: mov d7, v4.d[1]
+; CHECK-NEXT: mov d16, v0.d[1]
+; CHECK-NEXT: and v3.8b, v3.8b, v1.8b
+; CHECK-NEXT: and v2.8b, v2.8b, v1.8b
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: and v4.8b, v4.8b, v1.8b
+; CHECK-NEXT: and v5.8b, v5.8b, v1.8b
+; CHECK-NEXT: and v6.8b, v6.8b, v1.8b
+; CHECK-NEXT: and v7.8b, v7.8b, v1.8b
+; CHECK-NEXT: and v1.8b, v16.8b, v1.8b
; CHECK-NEXT: ushll v3.2d, v3.2s, #0
-; CHECK-NEXT: ushll v4.2d, v4.2s, #0
+; CHECK-NEXT: ushll v16.2d, v2.2s, #0
+; CHECK-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-NEXT: ushll v18.2d, v4.2s, #0
; CHECK-NEXT: ushll v5.2d, v5.2s, #0
; CHECK-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-NEXT: ushll v7.2d, v7.2s, #0
-; CHECK-NEXT: ushll v16.2d, v1.2s, #0
-; CHECK-NEXT: ucvtf v0.2d, v0.2d
-; CHECK-NEXT: ucvtf v1.2d, v2.2d
+; CHECK-NEXT: ushll v19.2d, v1.2s, #0
; CHECK-NEXT: ucvtf v2.2d, v3.2d
-; CHECK-NEXT: ucvtf v3.2d, v4.2d
-; CHECK-NEXT: ucvtf v4.2d, v5.2d
-; CHECK-NEXT: ucvtf v5.2d, v6.2d
-; CHECK-NEXT: ucvtf v6.2d, v7.2d
-; CHECK-NEXT: ucvtf v7.2d, v16.2d
+; CHECK-NEXT: ucvtf v0.2d, v16.2d
+; CHECK-NEXT: ucvtf v4.2d, v17.2d
+; CHECK-NEXT: ucvtf v3.2d, v5.2d
+; CHECK-NEXT: ucvtf v1.2d, v6.2d
+; CHECK-NEXT: ucvtf v6.2d, v18.2d
+; CHECK-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-NEXT: ucvtf v5.2d, v19.2d
; CHECK-NEXT: ret
%1 = uitofp <16 x i8> %a to <16 x double>
ret <16 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll
index 5ed427110ea2f..5e11c868b549b 100644
--- a/llvm/test/CodeGen/AArch64/vselect-ext.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll
@@ -222,14 +222,16 @@ define <3 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v3i16(<3 x i8>
; CHECK-NEXT: movi.2d v3, #0x0000ff000000ff
; CHECK-NEXT: Lloh1:
; CHECK-NEXT: ldr d2, [x8, lCPI9_0 at PAGEOFF]
-; CHECK-NEXT: mov.h v0[1], w1
-; CHECK-NEXT: mov.h v0[2], w2
-; CHECK-NEXT: ushll.4s v1, v0, #0
-; CHECK-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-NEXT: cmhi.4h v0, v0, v2
-; CHECK-NEXT: and.16b v1, v1, v3
-; CHECK-NEXT: sshll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: mov.16b v1, v0
+; CHECK-NEXT: mov.s v0[1], w1
+; CHECK-NEXT: mov.h v1[1], w1
+; CHECK-NEXT: mov.s v0[2], w2
+; CHECK-NEXT: mov.h v1[2], w2
+; CHECK-NEXT: and.16b v0, v0, v3
+; CHECK-NEXT: bic.4h v1, #255, lsl #8
+; CHECK-NEXT: cmhi.4h v1, v1, v2
+; CHECK-NEXT: sshll.4s v1, v1, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1
%ext = zext <3 x i8> %a to <3 x i32>
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index ea5a375c4cd8e..d1415b655cb5c 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1262,48 +1262,50 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
; CHECK-NEXT: ldr x9, [x0, x8]
; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: cmp x8, #128
-; CHECK-NEXT: ubfx x12, x9, #48, #4
-; CHECK-NEXT: lsr x10, x9, #52
-; CHECK-NEXT: ubfx x13, x9, #32, #4
-; CHECK-NEXT: ubfx w15, w9, #16, #4
-; CHECK-NEXT: lsr x11, x9, #36
-; CHECK-NEXT: lsr w14, w9, #20
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: fmov s1, w13
-; CHECK-NEXT: lsr w12, w9, #4
-; CHECK-NEXT: fmov s2, w15
-; CHECK-NEXT: mov.h v0[1], w10
-; CHECK-NEXT: and w10, w9, #0xf
-; CHECK-NEXT: mov.h v1[1], w11
-; CHECK-NEXT: fmov s3, w10
-; CHECK-NEXT: lsr x11, x9, #56
-; CHECK-NEXT: mov.h v2[1], w14
-; CHECK-NEXT: lsr x10, x9, #40
-; CHECK-NEXT: mov.h v3[1], w12
-; CHECK-NEXT: lsr w12, w9, #24
-; CHECK-NEXT: mov.h v0[2], w11
-; CHECK-NEXT: lsr w11, w9, #8
-; CHECK-NEXT: mov.h v1[2], w10
-; CHECK-NEXT: lsr x10, x9, #60
-; CHECK-NEXT: mov.h v2[2], w12
-; CHECK-NEXT: lsr x12, x9, #44
-; CHECK-NEXT: mov.h v3[2], w11
-; CHECK-NEXT: lsr w11, w9, #28
-; CHECK-NEXT: lsr w9, w9, #12
-; CHECK-NEXT: mov.h v0[3], w10
-; CHECK-NEXT: mov.h v1[3], w12
-; CHECK-NEXT: mov.h v2[3], w11
-; CHECK-NEXT: mov.h v3[3], w9
-; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: ubfx x10, x9, #32, #4
+; CHECK-NEXT: and w12, w9, #0xf
+; CHECK-NEXT: ubfx x11, x9, #36, #4
+; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #4, #4
+; CHECK-NEXT: mov.b v1[1], w10
+; CHECK-NEXT: ubfx w10, w9, #8, #4
+; CHECK-NEXT: mov.b v0[1], w11
+; CHECK-NEXT: ubfx x11, x9, #40, #4
+; CHECK-NEXT: mov.b v1[2], w10
+; CHECK-NEXT: ubfx w10, w9, #12, #4
+; CHECK-NEXT: mov.b v0[2], w11
+; CHECK-NEXT: ubfx x11, x9, #44, #4
+; CHECK-NEXT: mov.b v1[3], w10
+; CHECK-NEXT: ubfx w10, w9, #16, #4
+; CHECK-NEXT: mov.b v0[3], w11
+; CHECK-NEXT: ubfx x11, x9, #48, #4
+; CHECK-NEXT: mov.b v1[4], w10
+; CHECK-NEXT: ubfx w10, w9, #20, #4
+; CHECK-NEXT: mov.b v0[4], w11
+; CHECK-NEXT: ubfx x11, x9, #52, #4
+; CHECK-NEXT: mov.b v1[5], w10
+; CHECK-NEXT: ubfx w10, w9, #24, #4
+; CHECK-NEXT: mov.b v0[5], w11
+; CHECK-NEXT: ubfx x11, x9, #56, #4
+; CHECK-NEXT: mov.b v1[6], w10
+; CHECK-NEXT: mov.b v0[6], w11
+; CHECK-NEXT: lsr x11, x9, #60
+; CHECK-NEXT: lsr w9, w9, #28
+; CHECK-NEXT: mov.b v1[7], w9
+; CHECK-NEXT: mov.b v0[7], w11
+; CHECK-NEXT: ushll.8h v1, v1, #0
+; CHECK-NEXT: ushll.8h v0, v0, #0
+; CHECK-NEXT: ushll2.4s v3, v1, #0
; CHECK-NEXT: ushll.4s v1, v1, #0
-; CHECK-NEXT: ushll.4s v2, v2, #0
-; CHECK-NEXT: ushll.4s v3, v3, #0
-; CHECK-NEXT: and z0.s, z0.s, #0xf
+; CHECK-NEXT: ushll2.4s v2, v0, #0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and z3.s, z3.s, #0xf
; CHECK-NEXT: and z1.s, z1.s, #0xf
; CHECK-NEXT: and z2.s, z2.s, #0xf
-; CHECK-NEXT: and z3.s, z3.s, #0xf
-; CHECK-NEXT: stp q1, q0, [x1, #32]
-; CHECK-NEXT: stp q3, q2, [x1], #64
+; CHECK-NEXT: and z0.s, z0.s, #0xf
+; CHECK-NEXT: stp q0, q2, [x1, #32]
+; CHECK-NEXT: stp q1, q3, [x1], #64
; CHECK-NEXT: b.ne LBB13_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: ret
@@ -1316,53 +1318,56 @@ define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {
; CHECK-BE-NEXT: ldr x9, [x0, x8]
; CHECK-BE-NEXT: add x8, x8, #16
; CHECK-BE-NEXT: cmp x8, #128
-; CHECK-BE-NEXT: ubfx w11, w9, #12, #4
-; CHECK-BE-NEXT: lsr w14, w9, #28
-; CHECK-BE-NEXT: lsr w10, w9, #8
-; CHECK-BE-NEXT: ubfx x15, x9, #44, #4
-; CHECK-BE-NEXT: lsr w12, w9, #24
-; CHECK-BE-NEXT: lsr x13, x9, #40
-; CHECK-BE-NEXT: fmov s0, w11
-; CHECK-BE-NEXT: lsr x11, x9, #60
-; CHECK-BE-NEXT: fmov s1, w14
-; CHECK-BE-NEXT: fmov s2, w15
-; CHECK-BE-NEXT: fmov s3, w11
-; CHECK-BE-NEXT: lsr w11, w9, #20
-; CHECK-BE-NEXT: mov v0.h[1], w10
-; CHECK-BE-NEXT: lsr x10, x9, #56
-; CHECK-BE-NEXT: mov v1.h[1], w12
-; CHECK-BE-NEXT: lsr w12, w9, #4
-; CHECK-BE-NEXT: mov v2.h[1], w13
-; CHECK-BE-NEXT: mov v3.h[1], w10
-; CHECK-BE-NEXT: lsr x10, x9, #36
-; CHECK-BE-NEXT: mov v0.h[2], w12
-; CHECK-BE-NEXT: lsr x12, x9, #52
-; CHECK-BE-NEXT: mov v1.h[2], w11
-; CHECK-BE-NEXT: mov v2.h[2], w10
-; CHECK-BE-NEXT: lsr w10, w9, #16
-; CHECK-BE-NEXT: lsr x11, x9, #32
-; CHECK-BE-NEXT: mov v3.h[2], w12
-; CHECK-BE-NEXT: mov v0.h[3], w9
-; CHECK-BE-NEXT: lsr x9, x9, #48
-; CHECK-BE-NEXT: mov v1.h[3], w10
-; CHECK-BE-NEXT: mov v2.h[3], w11
-; CHECK-BE-NEXT: add x10, x1, #32
-; CHECK-BE-NEXT: mov v3.h[3], w9
+; CHECK-BE-NEXT: lsr w10, w9, #28
+; CHECK-BE-NEXT: lsr x12, x9, #60
+; CHECK-BE-NEXT: ubfx w11, w9, #24, #4
+; CHECK-BE-NEXT: ubfx x13, x9, #56, #4
+; CHECK-BE-NEXT: fmov s0, w10
+; CHECK-BE-NEXT: fmov s1, w12
+; CHECK-BE-NEXT: ubfx w10, w9, #20, #4
+; CHECK-BE-NEXT: mov v0.b[1], w11
+; CHECK-BE-NEXT: mov v1.b[1], w13
+; CHECK-BE-NEXT: ubfx x11, x9, #52, #4
+; CHECK-BE-NEXT: mov v0.b[2], w10
+; CHECK-BE-NEXT: mov v1.b[2], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #16, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #48, #4
+; CHECK-BE-NEXT: mov v0.b[3], w10
+; CHECK-BE-NEXT: mov v1.b[3], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #12, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #44, #4
+; CHECK-BE-NEXT: mov v0.b[4], w10
+; CHECK-BE-NEXT: mov v1.b[4], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #8, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #40, #4
+; CHECK-BE-NEXT: mov v0.b[5], w10
+; CHECK-BE-NEXT: mov v1.b[5], w11
+; CHECK-BE-NEXT: ubfx w10, w9, #4, #4
+; CHECK-BE-NEXT: ubfx x11, x9, #36, #4
+; CHECK-BE-NEXT: mov v0.b[6], w10
+; CHECK-BE-NEXT: mov v1.b[6], w11
+; CHECK-BE-NEXT: and w10, w9, #0xf
+; CHECK-BE-NEXT: ubfx x9, x9, #32, #4
+; CHECK-BE-NEXT: mov v0.b[7], w10
+; CHECK-BE-NEXT: mov v1.b[7], w9
; CHECK-BE-NEXT: add x9, x1, #48
-; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: add x10, x1, #16
+; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BE-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-BE-NEXT: ushll2 v3.4s, v1.8h, #0
; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-BE-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-BE-NEXT: and z0.s, z0.s, #0xf
-; CHECK-BE-NEXT: and z1.s, z1.s, #0xf
+; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-BE-NEXT: and z2.s, z2.s, #0xf
+; CHECK-BE-NEXT: and z1.s, z1.s, #0xf
; CHECK-BE-NEXT: and z3.s, z3.s, #0xf
-; CHECK-BE-NEXT: st1 { v0.4s }, [x9]
-; CHECK-BE-NEXT: add x9, x1, #16
-; CHECK-BE-NEXT: st1 { v1.4s }, [x10]
-; CHECK-BE-NEXT: st1 { v3.4s }, [x1]
-; CHECK-BE-NEXT: add x1, x1, #64
+; CHECK-BE-NEXT: and z0.s, z0.s, #0xf
; CHECK-BE-NEXT: st1 { v2.4s }, [x9]
+; CHECK-BE-NEXT: add x9, x1, #32
+; CHECK-BE-NEXT: st1 { v1.4s }, [x1]
+; CHECK-BE-NEXT: add x1, x1, #64
+; CHECK-BE-NEXT: st1 { v3.4s }, [x10]
+; CHECK-BE-NEXT: st1 { v0.4s }, [x9]
; CHECK-BE-NEXT: b.ne .LBB13_1
; CHECK-BE-NEXT: // %bb.2: // %exit
; CHECK-BE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll
index 9ec760aa57e27..8f02da9c78223 100644
--- a/llvm/test/CodeGen/AArch64/zext.ll
+++ b/llvm/test/CodeGen/AArch64/zext.ll
@@ -260,9 +260,8 @@ define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.2d, #0x0000ff000000ff
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -416,9 +415,8 @@ define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v0.s[2], w2
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -1044,39 +1042,33 @@ entry:
define <16 x i32> @zext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: zext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr w8, [sp, #32]
-; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr w10, [sp, #40]
-; CHECK-SD-NEXT: ldr w11, [sp, #8]
-; CHECK-SD-NEXT: fmov s2, w9
-; CHECK-SD-NEXT: fmov s3, w8
-; CHECK-SD-NEXT: ldr w8, [sp, #48]
-; CHECK-SD-NEXT: mov v0.h[1], w1
-; CHECK-SD-NEXT: ldr w9, [sp, #16]
+; CHECK-SD-NEXT: ldr s2, [sp]
+; CHECK-SD-NEXT: ldr s3, [sp, #32]
+; CHECK-SD-NEXT: add x8, sp, #8
+; CHECK-SD-NEXT: add x9, sp, #40
+; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #16
; CHECK-SD-NEXT: movi v4.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v1.h[1], w5
-; CHECK-SD-NEXT: mov v2.h[1], w11
-; CHECK-SD-NEXT: mov v3.h[1], w10
-; CHECK-SD-NEXT: mov v0.h[2], w2
-; CHECK-SD-NEXT: mov v1.h[2], w6
-; CHECK-SD-NEXT: mov v2.h[2], w9
-; CHECK-SD-NEXT: mov v3.h[2], w8
-; CHECK-SD-NEXT: ldr w8, [sp, #56]
-; CHECK-SD-NEXT: ldr w9, [sp, #24]
-; CHECK-SD-NEXT: mov v0.h[3], w3
-; CHECK-SD-NEXT: mov v1.h[3], w7
-; CHECK-SD-NEXT: mov v2.h[3], w9
-; CHECK-SD-NEXT: mov v3.h[3], w8
-; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-SD-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v1.s[1], w5
+; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #48
+; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #56
+; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v1.s[2], w6
+; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-SD-NEXT: mov v0.s[3], w3
+; CHECK-SD-NEXT: mov v1.s[3], w7
; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-SD-NEXT: and v1.16b, v1.16b, v4.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: zext_v16i10_v16i32:
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index 68ca43b05eaa6..bada3532502d8 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -4356,22 +4356,18 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; SKX-LABEL: mask64_insert:
; SKX: ## %bb.0:
-; SKX-NEXT: andb $1, %dil
-; SKX-NEXT: cmpb $1, %dil
-; SKX-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; SKX-NEXT: sbbq $-1, %rax
-; SKX-NEXT: kmovq %rax, %k0
-; SKX-NEXT: vpmovm2b %k0, %zmm0
+; SKX-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SKX-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; SKX-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
+; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; SKX-NEXT: retq
;
; AVX512BW-LABEL: mask64_insert:
; AVX512BW: ## %bb.0:
-; AVX512BW-NEXT: andb $1, %dil
-; AVX512BW-NEXT: cmpb $1, %dil
-; AVX512BW-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; AVX512BW-NEXT: sbbq $-1, %rax
-; AVX512BW-NEXT: kmovq %rax, %k0
-; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
+; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512BW-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
+; AVX512BW-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: mask64_insert:
@@ -4388,16 +4384,10 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; X86-LABEL: mask64_insert:
; X86: ## %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: andb $1, %al
-; X86-NEXT: cmpb $1, %al
-; X86-NEXT: movl $-131076, %eax ## imm = 0xFFFDFFFC
-; X86-NEXT: sbbl $-1, %eax
-; X86-NEXT: kmovd %eax, %k0
-; X86-NEXT: movl $-131075, %eax ## imm = 0xFFFDFFFD
-; X86-NEXT: kmovd %eax, %k1
-; X86-NEXT: kunpckdq %k0, %k1, %k0
-; X86-NEXT: vpmovm2b %k0, %zmm0
+; X86-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; X86-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X86-NEXT: vpinsrb $0, {{[0-9]+}}(%esp), %xmm0, %xmm1
+; X86-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; X86-NEXT: retl
%a_i = trunc i32 %a to i1
%maskv = insertelement <64 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
>From 719ef2ca606000dc2b5df0e8e897934b06f3a032 Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 09:43:34 -0700
Subject: [PATCH 3/5] revert/split the dagcombiner change
Change-Id: Id5224e69cfb74b5e92b60b071149390410097514
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 -
.../aarch64-neon-vector-insert-uaddlv.ll | 6 +-
.../CodeGen/AArch64/combine-storetomstore.ll | 60 ++++---
llvm/test/CodeGen/AArch64/neon-extadd.ll | 152 +++++++++-------
llvm/test/CodeGen/AArch64/sext.ll | 64 ++++---
.../AArch64/vec3-loads-ext-trunc-stores.ll | 168 ++++++++----------
llvm/test/CodeGen/AArch64/vec_uaddo.ll | 8 +-
llvm/test/CodeGen/AArch64/vec_umulo.ll | 12 +-
llvm/test/CodeGen/AArch64/vector-compress.ll | 7 +-
llvm/test/CodeGen/AArch64/vselect-ext.ll | 18 +-
llvm/test/CodeGen/AArch64/zext.ll | 60 ++++---
11 files changed, 299 insertions(+), 265 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 3f4442a7fcde8..595cc5d653de6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -16785,15 +16785,6 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
return SCC;
}
- // Build vector can be truncating. All operands have the same type. Check if
- // we can do the build vector in a wider type.
- // ty1 aext(ty2 build_vector()) -> ty1 build_vector() of wider type.
- if (N0.getOpcode() == ISD::BUILD_VECTOR &&
- VT.getScalarSizeInBits() <=
- N0.getOperand(0).getValueType().getScalarSizeInBits() &&
- (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)))
- return DAG.getBuildVector(VT, DL, N0->ops());
-
if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
return NewCtPop;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d552397..bbb9e98a39e31 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -281,7 +281,8 @@ define void @insert_vec_v16i8_uaddlv_from_v8i8(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.8b h1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.h v2[0], v1[0]
+; CHECK-NEXT: mov.b v2[0], v1[0]
+; CHECK-NEXT: zip1.8b v2, v2, v2
; CHECK-NEXT: bic.4h v2, #255, lsl #8
; CHECK-NEXT: ushll.4s v2, v2, #0
; CHECK-NEXT: ucvtf.4s v2, v2
@@ -457,7 +458,8 @@ define void @insert_vec_v16i8_uaddlv_from_v4i32(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4s d0, v0
; CHECK-NEXT: stp q2, q2, [x0, #32]
-; CHECK-NEXT: mov.h v1[0], v0[0]
+; CHECK-NEXT: mov.b v1[0], v0[0]
+; CHECK-NEXT: zip1.8b v1, v1, v1
; CHECK-NEXT: bic.4h v1, #255, lsl #8
; CHECK-NEXT: ushll.4s v1, v1, #0
; CHECK-NEXT: ucvtf.4s v1, v1
diff --git a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
index bdfd58787437e..e4b1965e4b76c 100644
--- a/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/AArch64/combine-storetomstore.ll
@@ -455,60 +455,64 @@ define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1>
; SVE-NEXT: fmov s4, w9
; SVE-NEXT: ldr w9, [sp, #8]
; SVE-NEXT: fmov s5, w11
-; SVE-NEXT: mov v7.h[1], w2
+; SVE-NEXT: mov v7.b[1], w2
; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
; SVE-NEXT: fmov s6, w9
; SVE-NEXT: ldr w9, [sp, #152]
-; SVE-NEXT: mov v4.h[1], w8
+; SVE-NEXT: mov v4.b[1], w8
; SVE-NEXT: ldr w8, [sp, #16]
-; SVE-NEXT: mov v5.h[1], w10
-; SVE-NEXT: mov v6.h[1], w8
+; SVE-NEXT: mov v5.b[1], w10
+; SVE-NEXT: mov v6.b[1], w8
; SVE-NEXT: ldr w8, [sp, #88]
-; SVE-NEXT: mov v7.h[2], w3
-; SVE-NEXT: mov v4.h[2], w8
+; SVE-NEXT: mov v7.b[2], w3
+; SVE-NEXT: mov v4.b[2], w8
; SVE-NEXT: ldr w8, [sp, #24]
-; SVE-NEXT: mov v5.h[2], w9
+; SVE-NEXT: mov v5.b[2], w9
; SVE-NEXT: ldr w9, [sp, #160]
-; SVE-NEXT: mov v6.h[2], w8
+; SVE-NEXT: mov v6.b[2], w8
; SVE-NEXT: ldr w8, [sp, #96]
-; SVE-NEXT: mov v7.h[3], w4
-; SVE-NEXT: mov v4.h[3], w8
+; SVE-NEXT: mov v7.b[3], w4
+; SVE-NEXT: mov v4.b[3], w8
; SVE-NEXT: ldr w8, [sp, #32]
-; SVE-NEXT: mov v5.h[3], w9
+; SVE-NEXT: mov v5.b[3], w9
; SVE-NEXT: ldr w9, [sp, #168]
-; SVE-NEXT: mov v6.h[3], w8
+; SVE-NEXT: mov v6.b[3], w8
; SVE-NEXT: ldr w8, [sp, #104]
-; SVE-NEXT: mov v7.h[4], w5
-; SVE-NEXT: mov v4.h[4], w8
+; SVE-NEXT: mov v7.b[4], w5
+; SVE-NEXT: mov v4.b[4], w8
; SVE-NEXT: ldr w8, [sp, #40]
-; SVE-NEXT: mov v5.h[4], w9
+; SVE-NEXT: mov v5.b[4], w9
; SVE-NEXT: ldr w9, [sp, #176]
-; SVE-NEXT: mov v6.h[4], w8
+; SVE-NEXT: mov v6.b[4], w8
; SVE-NEXT: ldr w8, [sp, #112]
-; SVE-NEXT: mov v7.h[5], w6
-; SVE-NEXT: mov v4.h[5], w8
+; SVE-NEXT: mov v7.b[5], w6
+; SVE-NEXT: mov v4.b[5], w8
; SVE-NEXT: ldr w8, [sp, #48]
-; SVE-NEXT: mov v5.h[5], w9
+; SVE-NEXT: mov v5.b[5], w9
; SVE-NEXT: ldr w9, [sp, #184]
-; SVE-NEXT: mov v6.h[5], w8
+; SVE-NEXT: mov v6.b[5], w8
; SVE-NEXT: ldr w8, [sp, #120]
-; SVE-NEXT: mov v7.h[6], w7
-; SVE-NEXT: mov v4.h[6], w8
+; SVE-NEXT: mov v7.b[6], w7
+; SVE-NEXT: mov v4.b[6], w8
; SVE-NEXT: ldr w8, [sp, #56]
-; SVE-NEXT: mov v5.h[6], w9
+; SVE-NEXT: mov v5.b[6], w9
; SVE-NEXT: ldr w9, [sp, #192]
-; SVE-NEXT: mov v6.h[6], w8
+; SVE-NEXT: mov v6.b[6], w8
; SVE-NEXT: ldr w8, [sp, #128]
-; SVE-NEXT: mov v4.h[7], w8
+; SVE-NEXT: mov v4.b[7], w8
; SVE-NEXT: ldr w8, [sp, #64]
-; SVE-NEXT: mov v5.h[7], w9
+; SVE-NEXT: mov v5.b[7], w9
; SVE-NEXT: ldr w9, [sp]
-; SVE-NEXT: mov v6.h[7], w8
+; SVE-NEXT: mov v6.b[7], w8
; SVE-NEXT: mov x8, #16 // =0x10
-; SVE-NEXT: mov v7.h[7], w9
+; SVE-NEXT: mov v7.b[7], w9
+; SVE-NEXT: ushll v4.8h, v4.8b, #0
+; SVE-NEXT: ushll v5.8h, v5.8b, #0
+; SVE-NEXT: ushll v6.8h, v6.8b, #0
+; SVE-NEXT: ushll v7.8h, v7.8b, #0
; SVE-NEXT: shl v4.8h, v4.8h, #15
; SVE-NEXT: shl v5.8h, v5.8h, #15
; SVE-NEXT: shl v6.8h, v6.8h, #15
diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll
index b48fb576ff1e0..627dab6402791 100644
--- a/llvm/test/CodeGen/AArch64/neon-extadd.ll
+++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll
@@ -1368,69 +1368,95 @@ entry:
define <16 x i32> @i12(<16 x i12> %s0, <16 x i12> %s1) {
; CHECK-SD-LABEL: i12:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr s2, [sp, #32]
-; CHECK-SD-NEXT: add x8, sp, #40
-; CHECK-SD-NEXT: fmov s0, w4
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: ldr s4, [sp]
-; CHECK-SD-NEXT: add x9, sp, #8
-; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #48
-; CHECK-SD-NEXT: ldr s6, [sp, #64]
-; CHECK-SD-NEXT: ld1 { v4.s }[1], [x9]
-; CHECK-SD-NEXT: mov v0.s[1], w5
-; CHECK-SD-NEXT: add x9, sp, #16
-; CHECK-SD-NEXT: mov v1.s[1], w1
-; CHECK-SD-NEXT: ldr s5, [sp, #96]
-; CHECK-SD-NEXT: add x11, sp, #72
-; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #56
-; CHECK-SD-NEXT: ldr s7, [sp, #160]
-; CHECK-SD-NEXT: ldr s16, [sp, #128]
-; CHECK-SD-NEXT: add x10, sp, #104
-; CHECK-SD-NEXT: ld1 { v4.s }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v6.s }[1], [x11]
-; CHECK-SD-NEXT: add x9, sp, #136
-; CHECK-SD-NEXT: ld1 { v5.s }[1], [x10]
-; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #168
-; CHECK-SD-NEXT: ld1 { v16.s }[1], [x9]
-; CHECK-SD-NEXT: ld1 { v7.s }[1], [x8]
-; CHECK-SD-NEXT: mov v0.s[2], w6
-; CHECK-SD-NEXT: mov v1.s[2], w2
-; CHECK-SD-NEXT: add x10, sp, #80
-; CHECK-SD-NEXT: add x9, sp, #112
-; CHECK-SD-NEXT: add x11, sp, #144
-; CHECK-SD-NEXT: ld1 { v6.s }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #176
-; CHECK-SD-NEXT: ld1 { v5.s }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v7.s }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v16.s }[2], [x11]
-; CHECK-SD-NEXT: add x8, sp, #24
-; CHECK-SD-NEXT: add x9, sp, #88
-; CHECK-SD-NEXT: movi v3.4s, #15, msl #8
-; CHECK-SD-NEXT: mov v0.s[3], w7
-; CHECK-SD-NEXT: mov v1.s[3], w3
-; CHECK-SD-NEXT: ld1 { v4.s }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #120
-; CHECK-SD-NEXT: ld1 { v6.s }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: add x10, sp, #152
-; CHECK-SD-NEXT: ld1 { v5.s }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v7.s }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v16.s }[3], [x10]
-; CHECK-SD-NEXT: and v17.16b, v0.16b, v3.16b
-; CHECK-SD-NEXT: and v0.16b, v1.16b, v3.16b
-; CHECK-SD-NEXT: and v1.16b, v2.16b, v3.16b
-; CHECK-SD-NEXT: and v2.16b, v4.16b, v3.16b
-; CHECK-SD-NEXT: and v4.16b, v6.16b, v3.16b
-; CHECK-SD-NEXT: and v5.16b, v5.16b, v3.16b
-; CHECK-SD-NEXT: and v6.16b, v7.16b, v3.16b
-; CHECK-SD-NEXT: and v3.16b, v16.16b, v3.16b
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-SD-NEXT: add v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT: add v3.4s, v1.4s, v6.4s
-; CHECK-SD-NEXT: add v1.4s, v17.4s, v5.4s
+; CHECK-SD-NEXT: str x23, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT: .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT: .cfi_offset w19, -8
+; CHECK-SD-NEXT: .cfi_offset w20, -16
+; CHECK-SD-NEXT: .cfi_offset w21, -24
+; CHECK-SD-NEXT: .cfi_offset w22, -32
+; CHECK-SD-NEXT: .cfi_offset w23, -48
+; CHECK-SD-NEXT: ldr w13, [sp, #112]
+; CHECK-SD-NEXT: ldr w14, [sp, #144]
+; CHECK-SD-NEXT: fmov s2, w4
+; CHECK-SD-NEXT: ldr w17, [sp, #176]
+; CHECK-SD-NEXT: ldr w19, [sp, #208]
+; CHECK-SD-NEXT: fmov s3, w0
+; CHECK-SD-NEXT: ldr w20, [sp, #80]
+; CHECK-SD-NEXT: ldr w21, [sp, #48]
+; CHECK-SD-NEXT: fmov s5, w13
+; CHECK-SD-NEXT: fmov s4, w19
+; CHECK-SD-NEXT: fmov s6, w17
+; CHECK-SD-NEXT: fmov s7, w14
+; CHECK-SD-NEXT: fmov s0, w20
+; CHECK-SD-NEXT: fmov s1, w21
+; CHECK-SD-NEXT: ldr w10, [sp, #120]
+; CHECK-SD-NEXT: ldr w11, [sp, #152]
+; CHECK-SD-NEXT: ldr w12, [sp, #184]
+; CHECK-SD-NEXT: ldr w15, [sp, #216]
+; CHECK-SD-NEXT: ldr w22, [sp, #88]
+; CHECK-SD-NEXT: ldr w23, [sp, #56]
+; CHECK-SD-NEXT: mov v2.h[1], w5
+; CHECK-SD-NEXT: mov v3.h[1], w1
+; CHECK-SD-NEXT: mov v5.h[1], w10
+; CHECK-SD-NEXT: mov v4.h[1], w15
+; CHECK-SD-NEXT: mov v0.h[1], w22
+; CHECK-SD-NEXT: mov v1.h[1], w23
+; CHECK-SD-NEXT: mov v6.h[1], w12
+; CHECK-SD-NEXT: mov v7.h[1], w11
+; CHECK-SD-NEXT: ldr w8, [sp, #128]
+; CHECK-SD-NEXT: ldr w9, [sp, #160]
+; CHECK-SD-NEXT: ldr w16, [sp, #64]
+; CHECK-SD-NEXT: ldr w18, [sp, #96]
+; CHECK-SD-NEXT: ldr w10, [sp, #192]
+; CHECK-SD-NEXT: ldr w11, [sp, #224]
+; CHECK-SD-NEXT: mov v2.h[2], w6
+; CHECK-SD-NEXT: mov v3.h[2], w2
+; CHECK-SD-NEXT: mov v0.h[2], w18
+; CHECK-SD-NEXT: mov v1.h[2], w16
+; CHECK-SD-NEXT: mov v5.h[2], w8
+; CHECK-SD-NEXT: mov v4.h[2], w11
+; CHECK-SD-NEXT: mov v6.h[2], w10
+; CHECK-SD-NEXT: mov v7.h[2], w9
+; CHECK-SD-NEXT: ldr w12, [sp, #72]
+; CHECK-SD-NEXT: ldr w13, [sp, #104]
+; CHECK-SD-NEXT: ldr w8, [sp, #136]
+; CHECK-SD-NEXT: ldr w9, [sp, #168]
+; CHECK-SD-NEXT: ldr w10, [sp, #200]
+; CHECK-SD-NEXT: ldr w11, [sp, #232]
+; CHECK-SD-NEXT: mov v0.h[3], w13
+; CHECK-SD-NEXT: mov v1.h[3], w12
+; CHECK-SD-NEXT: mov v2.h[3], w7
+; CHECK-SD-NEXT: mov v3.h[3], w3
+; CHECK-SD-NEXT: mov v5.h[3], w8
+; CHECK-SD-NEXT: mov v4.h[3], w11
+; CHECK-SD-NEXT: mov v6.h[3], w10
+; CHECK-SD-NEXT: mov v7.h[3], w9
+; CHECK-SD-NEXT: movi v16.4s, #15, msl #8
+; CHECK-SD-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-SD-NEXT: ushll v6.4s, v6.4h, #0
+; CHECK-SD-NEXT: ushll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT: and v17.16b, v0.16b, v16.16b
+; CHECK-SD-NEXT: and v18.16b, v1.16b, v16.16b
+; CHECK-SD-NEXT: and v1.16b, v2.16b, v16.16b
+; CHECK-SD-NEXT: and v0.16b, v3.16b, v16.16b
+; CHECK-SD-NEXT: and v2.16b, v5.16b, v16.16b
+; CHECK-SD-NEXT: and v3.16b, v4.16b, v16.16b
+; CHECK-SD-NEXT: and v4.16b, v6.16b, v16.16b
+; CHECK-SD-NEXT: and v5.16b, v7.16b, v16.16b
+; CHECK-SD-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: add v3.4s, v17.4s, v3.4s
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT: add v2.4s, v18.4s, v4.4s
+; CHECK-SD-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: i12:
diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll
index f1fae4aafd26a..278731ad0663a 100644
--- a/llvm/test/CodeGen/AArch64/sext.ll
+++ b/llvm/test/CodeGen/AArch64/sext.ll
@@ -239,8 +239,9 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD-LABEL: sext_v3i8_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24
; CHECK-SD-NEXT: ret
@@ -397,8 +398,9 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD-LABEL: sext_v3i10_v3i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
; CHECK-SD-NEXT: ret
@@ -1053,36 +1055,42 @@ entry:
define <16 x i32> @sext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: sext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr w8, [sp, #32]
+; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr s2, [sp]
-; CHECK-SD-NEXT: ldr s3, [sp, #32]
-; CHECK-SD-NEXT: add x8, sp, #8
-; CHECK-SD-NEXT: add x9, sp, #40
-; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #16
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v1.s[1], w5
-; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #48
-; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #24
-; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
-; CHECK-SD-NEXT: add x9, sp, #56
-; CHECK-SD-NEXT: mov v0.s[2], w2
-; CHECK-SD-NEXT: mov v1.s[2], w6
-; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
-; CHECK-SD-NEXT: mov v0.s[3], w3
-; CHECK-SD-NEXT: mov v1.s[3], w7
-; CHECK-SD-NEXT: shl v2.4s, v2.4s, #22
-; CHECK-SD-NEXT: shl v3.4s, v3.4s, #22
-; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT: ldr w10, [sp, #40]
+; CHECK-SD-NEXT: ldr w11, [sp, #8]
+; CHECK-SD-NEXT: fmov s2, w9
+; CHECK-SD-NEXT: fmov s3, w8
+; CHECK-SD-NEXT: ldr w8, [sp, #48]
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: ldr w9, [sp, #16]
+; CHECK-SD-NEXT: mov v1.h[1], w5
+; CHECK-SD-NEXT: mov v2.h[1], w11
+; CHECK-SD-NEXT: mov v3.h[1], w10
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: mov v1.h[2], w6
+; CHECK-SD-NEXT: mov v2.h[2], w9
+; CHECK-SD-NEXT: mov v3.h[2], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #56]
+; CHECK-SD-NEXT: ldr w9, [sp, #24]
+; CHECK-SD-NEXT: mov v0.h[3], w3
+; CHECK-SD-NEXT: mov v1.h[3], w7
+; CHECK-SD-NEXT: mov v2.h[3], w9
+; CHECK-SD-NEXT: mov v3.h[3], w8
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
; CHECK-SD-NEXT: shl v0.4s, v0.4s, #22
; CHECK-SD-NEXT: shl v1.4s, v1.4s, #22
-; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #22
+; CHECK-SD-NEXT: shl v2.4s, v2.4s, #22
+; CHECK-SD-NEXT: shl v3.4s, v3.4s, #22
; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #22
; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #22
+; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #22
+; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #22
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: sext_v16i10_v16i32:
diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 0b6e1dc7c0bbb..85af93eeadbdf 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -40,14 +40,12 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: umov.b w8, v1[1]
-; CHECK-NEXT: umov.b w9, v1[2]
-; CHECK-NEXT: mov.s v1[1], w8
-; CHECK-NEXT: mov.s v1[2], w9
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: zip1.8b v0, v0, v0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32:
@@ -55,16 +53,16 @@ define <4 x i32> @load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s1, [sp, #12]
-; BE-NEXT: rev32 v1.8b, v1.8b
-; BE-NEXT: ushll v1.8h, v1.8b, #0
-; BE-NEXT: umov w8, v1.h[1]
-; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: ldr s0, [sp, #12]
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: mov v1.s[2], w8
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: ushll v0.8h, v0.8b, #0
+; BE-NEXT: mov v0.h[1], v0.h[1]
+; BE-NEXT: mov v0.h[2], w8
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -80,14 +78,12 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #2]
; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: umov.b w8, v1[1]
-; CHECK-NEXT: umov.b w9, v1[2]
-; CHECK-NEXT: mov.s v1[1], w8
-; CHECK-NEXT: mov.s v1[2], w9
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: zip1.8b v0, v0, v0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_2:
@@ -95,16 +91,16 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s1, [sp, #12]
-; BE-NEXT: rev32 v1.8b, v1.8b
-; BE-NEXT: ushll v1.8h, v1.8b, #0
-; BE-NEXT: umov w8, v1.h[1]
-; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: ldr s0, [sp, #12]
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: mov v1.s[2], w8
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: ushll v0.8h, v0.8b, #0
+; BE-NEXT: mov v0.h[1], v0.h[1]
+; BE-NEXT: mov v0.h[2], w8
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -118,25 +114,21 @@ define <4 x i32> @load_v3i8_to_4xi32_align_2(ptr %src) {
define <4 x i32> @load_v3i8_to_4xi32_align_4(ptr %src) {
; CHECK-LABEL: load_v3i8_to_4xi32_align_4:
; CHECK: ; %bb.0:
-; CHECK-NEXT: ldr s1, [x0]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
-; CHECK-NEXT: umov.b w8, v1[1]
-; CHECK-NEXT: umov.b w9, v1[2]
-; CHECK-NEXT: mov.s v1[1], w8
-; CHECK-NEXT: mov.s v1[2], w9
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: ldr s0, [x0]
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
+; CHECK-NEXT: zip1.8b v0, v0, v0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_align_4:
; BE: // %bb.0:
-; BE-NEXT: ldr s1, [x0]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
-; BE-NEXT: rev32 v1.16b, v1.16b
-; BE-NEXT: umov w8, v1.b[1]
-; BE-NEXT: umov w9, v1.b[2]
-; BE-NEXT: mov v1.s[1], w8
-; BE-NEXT: mov v1.s[2], w9
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: ldr s0, [x0]
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: ret
@@ -151,14 +143,12 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #3]
; CHECK-NEXT: ldurh w9, [x0, #1]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: umov.b w8, v1[1]
-; CHECK-NEXT: umov.b w9, v1[2]
-; CHECK-NEXT: mov.s v1[1], w8
-; CHECK-NEXT: mov.s v1[2], w9
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: zip1.8b v0, v0, v0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_1:
@@ -166,16 +156,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_1(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #1]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s1, [sp, #12]
-; BE-NEXT: rev32 v1.8b, v1.8b
-; BE-NEXT: ushll v1.8h, v1.8b, #0
-; BE-NEXT: umov w8, v1.h[1]
-; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: ldr s0, [sp, #12]
; BE-NEXT: ldrsb w8, [x0, #3]
-; BE-NEXT: mov v1.s[2], w8
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: ushll v0.8h, v0.8b, #0
+; BE-NEXT: mov v0.h[1], v0.h[1]
+; BE-NEXT: mov v0.h[2], w8
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -192,14 +182,12 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; CHECK: ; %bb.0:
; CHECK-NEXT: ldrb w8, [x0, #5]
; CHECK-NEXT: ldurh w9, [x0, #3]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
; CHECK-NEXT: orr w8, w9, w8, lsl #16
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: umov.b w8, v1[1]
-; CHECK-NEXT: umov.b w9, v1[2]
-; CHECK-NEXT: mov.s v1[1], w8
-; CHECK-NEXT: mov.s v1[2], w9
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: zip1.8b v0, v0, v0
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: ret
;
; BE-LABEL: load_v3i8_to_4xi32_const_offset_3:
@@ -207,16 +195,16 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldurh w8, [x0, #3]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s1, [sp, #12]
-; BE-NEXT: rev32 v1.8b, v1.8b
-; BE-NEXT: ushll v1.8h, v1.8b, #0
-; BE-NEXT: umov w8, v1.h[1]
-; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: ldr s0, [sp, #12]
; BE-NEXT: ldrsb w8, [x0, #5]
-; BE-NEXT: mov v1.s[2], w8
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: ushll v0.8h, v0.8b, #0
+; BE-NEXT: mov v0.h[1], v0.h[1]
+; BE-NEXT: mov v0.h[2], w8
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
@@ -234,15 +222,15 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: movi.2d v0, #0x0000ff000000ff
+; CHECK-NEXT: movi.2d v1, #0x0000ff000000ff
; CHECK-NEXT: strh w8, [sp, #12]
-; CHECK-NEXT: ldr s1, [sp, #12]
-; CHECK-NEXT: ushll.8h v1, v1, #0
-; CHECK-NEXT: umov.h w8, v1[1]
-; CHECK-NEXT: mov.s v1[1], w8
+; CHECK-NEXT: ldr s0, [sp, #12]
; CHECK-NEXT: ldrsb w8, [x0, #2]
-; CHECK-NEXT: mov.s v1[2], w8
-; CHECK-NEXT: and.16b v0, v1, v0
+; CHECK-NEXT: ushll.8h v0, v0, #0
+; CHECK-NEXT: mov.h v0[1], v0[1]
+; CHECK-NEXT: mov.h v0[2], w8
+; CHECK-NEXT: ushll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v0, v1
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: ret
;
@@ -251,16 +239,16 @@ define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
; BE-NEXT: sub sp, sp, #16
; BE-NEXT: .cfi_def_cfa_offset 16
; BE-NEXT: ldrh w8, [x0]
-; BE-NEXT: movi v0.2d, #0x0000ff000000ff
+; BE-NEXT: movi v1.2d, #0x0000ff000000ff
; BE-NEXT: strh w8, [sp, #12]
-; BE-NEXT: ldr s1, [sp, #12]
-; BE-NEXT: rev32 v1.8b, v1.8b
-; BE-NEXT: ushll v1.8h, v1.8b, #0
-; BE-NEXT: umov w8, v1.h[1]
-; BE-NEXT: mov v1.s[1], w8
+; BE-NEXT: ldr s0, [sp, #12]
; BE-NEXT: ldrsb w8, [x0, #2]
-; BE-NEXT: mov v1.s[2], w8
-; BE-NEXT: and v0.16b, v1.16b, v0.16b
+; BE-NEXT: rev32 v0.8b, v0.8b
+; BE-NEXT: ushll v0.8h, v0.8b, #0
+; BE-NEXT: mov v0.h[1], v0.h[1]
+; BE-NEXT: mov v0.h[2], w8
+; BE-NEXT: ushll v0.4s, v0.4h, #0
+; BE-NEXT: and v0.16b, v0.16b, v1.16b
; BE-NEXT: rev64 v0.4s, v0.4s
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; BE-NEXT: add sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index a1766dfef9465..c87c82c3e3672 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -170,8 +170,12 @@ define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK-NEXT: add v2.8h, v0.8h, v1.8h
; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h
; CHECK-NEXT: str q2, [x0]
-; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
-; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: shl v1.4s, v1.4s, #31
+; CHECK-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index 66757b6f7cf51..319348010ad06 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -197,12 +197,16 @@ define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
+; CHECK-NEXT: mul v4.8h, v0.8h, v1.8h
; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: mul v3.8h, v0.8h, v1.8h
+; CHECK-NEXT: str q4, [x0]
; CHECK-NEXT: cmtst v2.8h, v2.8h, v2.8h
-; CHECK-NEXT: str q3, [x0]
-; CHECK-NEXT: sshll2 v1.4s, v2.8h, #0
-; CHECK-NEXT: sshll v0.4s, v2.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0
+; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: shl v2.4s, v2.4s, #31
+; CHECK-NEXT: cmlt v1.4s, v3.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v2.4s, #0
; CHECK-NEXT: ret
%t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
%val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/AArch64/vector-compress.ll b/llvm/test/CodeGen/AArch64/vector-compress.ll
index 74f38884fe9f6..55c343164a1b8 100644
--- a/llvm/test/CodeGen/AArch64/vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/vector-compress.ll
@@ -413,9 +413,10 @@ define <3 x i32> @test_compress_narrow(<3 x i32> %vec, <3 x i1> %mask) {
; CHECK-NEXT: movi.2d v1, #0000000000000000
; CHECK-NEXT: mov x11, sp
; CHECK-NEXT: str s0, [sp]
-; CHECK-NEXT: mov.s v1[0], w0
-; CHECK-NEXT: mov.s v1[1], w1
-; CHECK-NEXT: mov.s v1[2], w2
+; CHECK-NEXT: mov.h v1[0], w0
+; CHECK-NEXT: mov.h v1[1], w1
+; CHECK-NEXT: mov.h v1[2], w2
+; CHECK-NEXT: ushll.4s v1, v1, #0
; CHECK-NEXT: shl.4s v1, v1, #31
; CHECK-NEXT: cmlt.4s v1, v1, #0
; CHECK-NEXT: mov.s w8, v1[1]
diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll
index 5e11c868b549b..5ed427110ea2f 100644
--- a/llvm/test/CodeGen/AArch64/vselect-ext.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll
@@ -222,16 +222,14 @@ define <3 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v3i16(<3 x i8>
; CHECK-NEXT: movi.2d v3, #0x0000ff000000ff
; CHECK-NEXT: Lloh1:
; CHECK-NEXT: ldr d2, [x8, lCPI9_0 at PAGEOFF]
-; CHECK-NEXT: mov.16b v1, v0
-; CHECK-NEXT: mov.s v0[1], w1
-; CHECK-NEXT: mov.h v1[1], w1
-; CHECK-NEXT: mov.s v0[2], w2
-; CHECK-NEXT: mov.h v1[2], w2
-; CHECK-NEXT: and.16b v0, v0, v3
-; CHECK-NEXT: bic.4h v1, #255, lsl #8
-; CHECK-NEXT: cmhi.4h v1, v1, v2
-; CHECK-NEXT: sshll.4s v1, v1, #0
-; CHECK-NEXT: and.16b v0, v0, v1
+; CHECK-NEXT: mov.h v0[1], w1
+; CHECK-NEXT: mov.h v0[2], w2
+; CHECK-NEXT: ushll.4s v1, v0, #0
+; CHECK-NEXT: bic.4h v0, #255, lsl #8
+; CHECK-NEXT: cmhi.4h v0, v0, v2
+; CHECK-NEXT: and.16b v1, v1, v3
+; CHECK-NEXT: sshll.4s v0, v0, #0
+; CHECK-NEXT: and.16b v0, v1, v0
; CHECK-NEXT: ret
; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1
%ext = zext <3 x i8> %a to <3 x i32>
diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll
index 8f02da9c78223..9ec760aa57e27 100644
--- a/llvm/test/CodeGen/AArch64/zext.ll
+++ b/llvm/test/CodeGen/AArch64/zext.ll
@@ -260,8 +260,9 @@ define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.2d, #0x0000ff000000ff
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -415,8 +416,9 @@ define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: movi v1.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT: ret
;
@@ -1042,33 +1044,39 @@ entry:
define <16 x i32> @zext_v16i10_v16i32(<16 x i10> %a) {
; CHECK-SD-LABEL: zext_v16i10_v16i32:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr w8, [sp, #32]
+; CHECK-SD-NEXT: ldr w9, [sp]
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: ldr s2, [sp]
-; CHECK-SD-NEXT: ldr s3, [sp, #32]
-; CHECK-SD-NEXT: add x8, sp, #8
-; CHECK-SD-NEXT: add x9, sp, #40
-; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #16
+; CHECK-SD-NEXT: ldr w10, [sp, #40]
+; CHECK-SD-NEXT: ldr w11, [sp, #8]
+; CHECK-SD-NEXT: fmov s2, w9
+; CHECK-SD-NEXT: fmov s3, w8
+; CHECK-SD-NEXT: ldr w8, [sp, #48]
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: ldr w9, [sp, #16]
; CHECK-SD-NEXT: movi v4.4s, #3, msl #8
-; CHECK-SD-NEXT: mov v0.s[1], w1
-; CHECK-SD-NEXT: mov v1.s[1], w5
-; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #48
-; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #24
-; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9]
-; CHECK-SD-NEXT: add x9, sp, #56
-; CHECK-SD-NEXT: mov v0.s[2], w2
-; CHECK-SD-NEXT: mov v1.s[2], w6
-; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v3.s }[3], [x9]
-; CHECK-SD-NEXT: mov v0.s[3], w3
-; CHECK-SD-NEXT: mov v1.s[3], w7
-; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b
-; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-SD-NEXT: mov v1.h[1], w5
+; CHECK-SD-NEXT: mov v2.h[1], w11
+; CHECK-SD-NEXT: mov v3.h[1], w10
+; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: mov v1.h[2], w6
+; CHECK-SD-NEXT: mov v2.h[2], w9
+; CHECK-SD-NEXT: mov v3.h[2], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #56]
+; CHECK-SD-NEXT: ldr w9, [sp, #24]
+; CHECK-SD-NEXT: mov v0.h[3], w3
+; CHECK-SD-NEXT: mov v1.h[3], w7
+; CHECK-SD-NEXT: mov v2.h[3], w9
+; CHECK-SD-NEXT: mov v3.h[3], w8
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
; CHECK-SD-NEXT: and v0.16b, v0.16b, v4.16b
; CHECK-SD-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: zext_v16i10_v16i32:
>From 1f67706f5861c41a4e25c3fcdef4174f9fa729be Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 09:56:02 -0700
Subject: [PATCH 4/5] forget to update x86 test
Change-Id: I2d45995bd396e353816df6a654e00ed953676d6f
---
llvm/test/CodeGen/X86/avx512-mask-op.ll | 34 ++++++++++++++++---------
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index bada3532502d8..68ca43b05eaa6 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -4356,18 +4356,22 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; SKX-LABEL: mask64_insert:
; SKX: ## %bb.0:
-; SKX-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SKX-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; SKX-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
-; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; SKX-NEXT: andb $1, %dil
+; SKX-NEXT: cmpb $1, %dil
+; SKX-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; SKX-NEXT: sbbq $-1, %rax
+; SKX-NEXT: kmovq %rax, %k0
+; SKX-NEXT: vpmovm2b %k0, %zmm0
; SKX-NEXT: retq
;
; AVX512BW-LABEL: mask64_insert:
; AVX512BW: ## %bb.0:
-; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512BW-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT: vpinsrb $0, %edi, %xmm0, %xmm1
-; AVX512BW-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: andb $1, %dil
+; AVX512BW-NEXT: cmpb $1, %dil
+; AVX512BW-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; AVX512BW-NEXT: sbbq $-1, %rax
+; AVX512BW-NEXT: kmovq %rax, %k0
+; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: mask64_insert:
@@ -4384,10 +4388,16 @@ define <64 x i1> @mask64_insert(i32 %a) {
;
; X86-LABEL: mask64_insert:
; X86: ## %bb.0:
-; X86-NEXT: vbroadcasti32x4 {{.*#+}} zmm0 = [1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; X86-NEXT: ## zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; X86-NEXT: vpinsrb $0, {{[0-9]+}}(%esp), %xmm0, %xmm1
-; X86-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andb $1, %al
+; X86-NEXT: cmpb $1, %al
+; X86-NEXT: movl $-131076, %eax ## imm = 0xFFFDFFFC
+; X86-NEXT: sbbl $-1, %eax
+; X86-NEXT: kmovd %eax, %k0
+; X86-NEXT: movl $-131075, %eax ## imm = 0xFFFDFFFD
+; X86-NEXT: kmovd %eax, %k1
+; X86-NEXT: kunpckdq %k0, %k1, %k0
+; X86-NEXT: vpmovm2b %k0, %zmm0
; X86-NEXT: retl
%a_i = trunc i32 %a to i1
%maskv = insertelement <64 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
>From 65b24aa738b4b2b5eb05e1609453b48733b50fbc Mon Sep 17 00:00:00 2001
From: Usman Nadeem <mnadeem at qti.qualcomm.com>
Date: Thu, 13 Aug 2026 14:00:07 -0700
Subject: [PATCH 5/5] test update after rebase
Change-Id: I54ee1e1f225f54e3df0e07d07eef5314c568e174
---
llvm/test/CodeGen/AArch64/neon-extadd.ll | 144 +++++++++++------------
1 file changed, 66 insertions(+), 78 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll
index 627dab6402791..01fdfa1ef2eb8 100644
--- a/llvm/test/CodeGen/AArch64/neon-extadd.ll
+++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll
@@ -1368,95 +1368,83 @@ entry:
define <16 x i32> @i12(<16 x i12> %s0, <16 x i12> %s1) {
; CHECK-SD-LABEL: i12:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: str x23, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-SD-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT: .cfi_offset w19, -8
-; CHECK-SD-NEXT: .cfi_offset w20, -16
-; CHECK-SD-NEXT: .cfi_offset w21, -24
-; CHECK-SD-NEXT: .cfi_offset w22, -32
-; CHECK-SD-NEXT: .cfi_offset w23, -48
-; CHECK-SD-NEXT: ldr w13, [sp, #112]
-; CHECK-SD-NEXT: ldr w14, [sp, #144]
-; CHECK-SD-NEXT: fmov s2, w4
-; CHECK-SD-NEXT: ldr w17, [sp, #176]
-; CHECK-SD-NEXT: ldr w19, [sp, #208]
+; CHECK-SD-NEXT: ldr w8, [sp, #32]
+; CHECK-SD-NEXT: ldr w9, [sp, #40]
+; CHECK-SD-NEXT: fmov s1, w4
+; CHECK-SD-NEXT: ldr w10, [sp, #128]
; CHECK-SD-NEXT: fmov s3, w0
-; CHECK-SD-NEXT: ldr w20, [sp, #80]
-; CHECK-SD-NEXT: ldr w21, [sp, #48]
-; CHECK-SD-NEXT: fmov s5, w13
-; CHECK-SD-NEXT: fmov s4, w19
-; CHECK-SD-NEXT: fmov s6, w17
-; CHECK-SD-NEXT: fmov s7, w14
-; CHECK-SD-NEXT: fmov s0, w20
-; CHECK-SD-NEXT: fmov s1, w21
-; CHECK-SD-NEXT: ldr w10, [sp, #120]
-; CHECK-SD-NEXT: ldr w11, [sp, #152]
-; CHECK-SD-NEXT: ldr w12, [sp, #184]
-; CHECK-SD-NEXT: ldr w15, [sp, #216]
-; CHECK-SD-NEXT: ldr w22, [sp, #88]
-; CHECK-SD-NEXT: ldr w23, [sp, #56]
-; CHECK-SD-NEXT: mov v2.h[1], w5
+; CHECK-SD-NEXT: ldr w11, [sp, #72]
+; CHECK-SD-NEXT: fmov s0, w8
+; CHECK-SD-NEXT: ldr w8, [sp]
+; CHECK-SD-NEXT: movi v16.4s, #15, msl #8
+; CHECK-SD-NEXT: fmov s5, w10
+; CHECK-SD-NEXT: ldr w10, [sp, #104]
+; CHECK-SD-NEXT: mov v1.h[1], w5
+; CHECK-SD-NEXT: fmov s2, w8
+; CHECK-SD-NEXT: ldr w8, [sp, #8]
; CHECK-SD-NEXT: mov v3.h[1], w1
-; CHECK-SD-NEXT: mov v5.h[1], w10
-; CHECK-SD-NEXT: mov v4.h[1], w15
-; CHECK-SD-NEXT: mov v0.h[1], w22
-; CHECK-SD-NEXT: mov v1.h[1], w23
-; CHECK-SD-NEXT: mov v6.h[1], w12
-; CHECK-SD-NEXT: mov v7.h[1], w11
-; CHECK-SD-NEXT: ldr w8, [sp, #128]
-; CHECK-SD-NEXT: ldr w9, [sp, #160]
-; CHECK-SD-NEXT: ldr w16, [sp, #64]
-; CHECK-SD-NEXT: ldr w18, [sp, #96]
-; CHECK-SD-NEXT: ldr w10, [sp, #192]
-; CHECK-SD-NEXT: ldr w11, [sp, #224]
-; CHECK-SD-NEXT: mov v2.h[2], w6
+; CHECK-SD-NEXT: mov v0.h[1], w9
+; CHECK-SD-NEXT: ldr w9, [sp, #48]
+; CHECK-SD-NEXT: mov v2.h[1], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #160]
+; CHECK-SD-NEXT: mov v1.h[2], w6
; CHECK-SD-NEXT: mov v3.h[2], w2
-; CHECK-SD-NEXT: mov v0.h[2], w18
-; CHECK-SD-NEXT: mov v1.h[2], w16
-; CHECK-SD-NEXT: mov v5.h[2], w8
-; CHECK-SD-NEXT: mov v4.h[2], w11
-; CHECK-SD-NEXT: mov v6.h[2], w10
-; CHECK-SD-NEXT: mov v7.h[2], w9
-; CHECK-SD-NEXT: ldr w12, [sp, #72]
-; CHECK-SD-NEXT: ldr w13, [sp, #104]
-; CHECK-SD-NEXT: ldr w8, [sp, #136]
+; CHECK-SD-NEXT: fmov s4, w8
+; CHECK-SD-NEXT: mov v0.h[2], w9
; CHECK-SD-NEXT: ldr w9, [sp, #168]
-; CHECK-SD-NEXT: ldr w10, [sp, #200]
-; CHECK-SD-NEXT: ldr w11, [sp, #232]
-; CHECK-SD-NEXT: mov v0.h[3], w13
-; CHECK-SD-NEXT: mov v1.h[3], w12
-; CHECK-SD-NEXT: mov v2.h[3], w7
+; CHECK-SD-NEXT: ldr w8, [sp, #64]
+; CHECK-SD-NEXT: mov v1.h[3], w7
+; CHECK-SD-NEXT: mov v4.h[1], w9
+; CHECK-SD-NEXT: ldr w9, [sp, #96]
+; CHECK-SD-NEXT: fmov s6, w8
+; CHECK-SD-NEXT: ldr w8, [sp, #136]
; CHECK-SD-NEXT: mov v3.h[3], w3
-; CHECK-SD-NEXT: mov v5.h[3], w8
-; CHECK-SD-NEXT: mov v4.h[3], w11
+; CHECK-SD-NEXT: fmov s7, w9
+; CHECK-SD-NEXT: ldr w9, [sp, #16]
+; CHECK-SD-NEXT: mov v6.h[1], w11
+; CHECK-SD-NEXT: mov v5.h[1], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #80]
+; CHECK-SD-NEXT: mov v2.h[2], w9
+; CHECK-SD-NEXT: ldr w9, [sp, #176]
+; CHECK-SD-NEXT: ldr w11, [sp, #144]
+; CHECK-SD-NEXT: mov v7.h[1], w10
+; CHECK-SD-NEXT: ldr w10, [sp, #112]
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: mov v4.h[2], w9
+; CHECK-SD-NEXT: ldr w9, [sp, #56]
+; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT: mov v6.h[2], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #24]
+; CHECK-SD-NEXT: mov v5.h[2], w11
+; CHECK-SD-NEXT: ldr w11, [sp, #120]
+; CHECK-SD-NEXT: mov v0.h[3], w9
+; CHECK-SD-NEXT: ldr w9, [sp, #152]
+; CHECK-SD-NEXT: mov v7.h[2], w10
+; CHECK-SD-NEXT: ldr w10, [sp, #88]
+; CHECK-SD-NEXT: mov v2.h[3], w8
+; CHECK-SD-NEXT: ldr w8, [sp, #184]
+; CHECK-SD-NEXT: and v1.16b, v1.16b, v16.16b
; CHECK-SD-NEXT: mov v6.h[3], w10
-; CHECK-SD-NEXT: mov v7.h[3], w9
-; CHECK-SD-NEXT: movi v16.4s, #15, msl #8
-; CHECK-SD-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT: mov v5.h[3], w9
+; CHECK-SD-NEXT: mov v4.h[3], w8
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: mov v7.h[3], w11
; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-SD-NEXT: ushll v4.4s, v4.4h, #0
; CHECK-SD-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT: ushll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT: ushll v5.4s, v5.4h, #0
; CHECK-SD-NEXT: and v17.16b, v0.16b, v16.16b
-; CHECK-SD-NEXT: and v18.16b, v1.16b, v16.16b
-; CHECK-SD-NEXT: and v1.16b, v2.16b, v16.16b
+; CHECK-SD-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-SD-NEXT: and v2.16b, v2.16b, v16.16b
; CHECK-SD-NEXT: and v0.16b, v3.16b, v16.16b
-; CHECK-SD-NEXT: and v2.16b, v5.16b, v16.16b
-; CHECK-SD-NEXT: and v3.16b, v4.16b, v16.16b
-; CHECK-SD-NEXT: and v4.16b, v6.16b, v16.16b
-; CHECK-SD-NEXT: and v5.16b, v7.16b, v16.16b
-; CHECK-SD-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-SD-NEXT: add v3.4s, v17.4s, v3.4s
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
-; CHECK-SD-NEXT: add v2.4s, v18.4s, v4.4s
-; CHECK-SD-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT: ushll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT: and v3.16b, v6.16b, v16.16b
+; CHECK-SD-NEXT: and v5.16b, v5.16b, v16.16b
+; CHECK-SD-NEXT: and v4.16b, v4.16b, v16.16b
+; CHECK-SD-NEXT: and v6.16b, v7.16b, v16.16b
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT: add v2.4s, v2.4s, v5.4s
+; CHECK-SD-NEXT: add v3.4s, v17.4s, v4.4s
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v6.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: i12:
More information about the llvm-commits
mailing list