[llvm] Use (de)interleaving intrinsics for fixed-length vectors (PR #202583)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 00:11:30 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/202583
>From 06b0bbc33cdb0500301c52cb5b4a7e05d6783026 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Tue, 28 Jul 2026 07:58:49 +0100
Subject: [PATCH 1/3] [CodeGen][AArch64] Expand unsupported interleave to
shuffles
---
llvm/include/llvm/CodeGen/TargetLowering.h | 8 +
.../SelectionDAG/LegalizeVectorOps.cpp | 90 ++
.../SelectionDAG/SelectionDAGBuilder.cpp | 22 -
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 3 +
llvm/lib/Target/RISCV/RISCVISelLowering.h | 3 +
.../aarch64-interleaved-access-w-undef.ll | 40 +-
.../AArch64/aarch64-interleaved-ld-combine.ll | 71 +-
llvm/test/CodeGen/AArch64/binopshuffles.ll | 73 +-
.../complex-deinterleaving-multiuses.ll | 288 ++-----
.../complex-deinterleaving-reductions.ll | 16 +-
.../AArch64/fixed-deinterleave-intrinsics.ll | 816 +++++-------------
.../AArch64/fixed-length-sve-interleave.ll | 19 +-
.../AArch64/fixed-vector-deinterleave.ll | 63 +-
.../fixed-vector-interleave-odd-factor.ll | 131 +++
.../AArch64/fixed-vector-interleave.ll | 50 +-
.../fixed_masked_deinterleaved_loads.ll | 28 +-
.../fixed_masked_interleaved_stores.ll | 24 +-
.../CodeGen/AArch64/interleaved-accesses.ll | 61 +-
.../AArch64/nontemporal-load-interleaved.ll | 522 ++++-------
.../nontemporal-store-interleaved-optsize.ll | 183 ++--
.../AArch64/nontemporal-store-interleaved.ll | 644 +++++---------
.../AArch64/sve-fixed-length-shuffles.ll | 26 +-
.../AArch64/sve-interleaved-accesses.ll | 93 +-
...-streaming-mode-fixed-length-interleave.ll | 128 +--
...-streaming-mode-fixed-length-ld2-alloca.ll | 8 +-
...sve-streaming-mode-fixed-length-shuffle.ll | 15 +-
llvm/test/CodeGen/AArch64/tbl-loops.ll | 229 ++---
.../AArch64/vector-deinterleave-load.ll | 39 +-
.../AArch64/vector-interleave-store.ll | 29 +-
llvm/test/CodeGen/AArch64/vldn_shuffle.ll | 243 ++----
.../X86/vector-interleave-intrinsics.ll | 576 +++++++++++++
31 files changed, 1921 insertions(+), 2620 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/fixed-vector-interleave-odd-factor.ll
create mode 100644 llvm/test/CodeGen/X86/vector-interleave-intrinsics.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9a525d69b3ee8..ca9792d6ca2d4 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3335,6 +3335,14 @@ class LLVM_ABI TargetLoweringBase {
/// Default to be the minimum interleave factor: 2.
virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
+ /// Return true if the target lowers ISD::VECTOR_INTERLEAVE and
+ /// ISD::VECTOR_DEINTERLEAVE of the given Factor and VecTy
+ /// itself.
+ virtual bool isInterleaveIntrinsicSupported(unsigned /*Factor*/,
+ EVT /*VecTy*/) const {
+ return false;
+ }
+
/// Lower an interleaved load to target specific intrinsics. Return
/// true on success.
///
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7b97c41ac2dbc..0e2baac46fa5c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -95,6 +95,14 @@ class VectorLegalizer {
/// operations to legalize them.
void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
+ /// Expand a VECTOR_INTERLEAVE into shuffles.
+ void ExpandVectorInterleaveToShuffles(SDNode *Node,
+ SmallVectorImpl<SDValue> &Results);
+
+ /// Expand a VECTOR_DEINTERLEAVE into shuffles.
+ void ExpandVectorDeinterleaveToShuffles(SDNode *Node,
+ SmallVectorImpl<SDValue> &Results);
+
/// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
/// FP_TO_SINT isn't legal.
void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
@@ -491,6 +499,18 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::MASKED_SREM:
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
+ case ISD::VECTOR_INTERLEAVE:
+ case ISD::VECTOR_DEINTERLEAVE: {
+ EVT VT = Node->getValueType(0);
+ unsigned Factor = Node->getNumOperands();
+ Action = TLI.getOperationAction(Node->getOpcode(), VT);
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, VT))
+ Action = TargetLowering::Expand;
+ else if (Action == TargetLowering::Custom)
+ // Leave the node untouched for LegalizeDAG.
+ Action = TargetLowering::Legal;
+ break;
+ }
case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
@@ -967,6 +987,70 @@ SDValue VectorLegalizer::ExpandStore(SDNode *N) {
return TF;
}
+void VectorLegalizer::ExpandVectorInterleaveToShuffles(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
+ EVT VT = Node->getValueType(0);
+ if (!VT.isFixedLengthVector())
+ return;
+
+ unsigned Factor = Node->getNumOperands();
+ unsigned NumElts = VT.getVectorNumElements();
+ SDLoc DL(Node);
+
+ // The interleaved value is the concatenation of the results, so build it as
+ // one wide shuffle and split that back up into one vector per factor.
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ NumElts * Factor);
+ SmallVector<SDValue, 8> InVecs(Node->op_values());
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, InVecs);
+ SDValue Interleaved =
+ DAG.getVectorShuffle(WideVT, DL, Concat, DAG.getUNDEF(WideVT),
+ createInterleaveMask(NumElts, Factor));
+
+ for (unsigned I = 0; I != Factor; ++I)
+ Results.push_back(
+ DAG.getExtractSubvector(DL, VT, Interleaved, I * NumElts));
+}
+
+void VectorLegalizer::ExpandVectorDeinterleaveToShuffles(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
+ EVT VT = Node->getValueType(0);
+ if (!VT.isFixedLengthVector())
+ return;
+
+ unsigned Factor = Node->getNumOperands();
+ unsigned NumElts = VT.getVectorNumElements();
+ SDLoc DL(Node);
+
+ // A factor of two is a plain two-input shuffle per result.
+ if (Factor == 2) {
+ for (unsigned I = 0; I != Factor; ++I)
+ Results.push_back(
+ DAG.getVectorShuffle(VT, DL, Node->getOperand(0), Node->getOperand(1),
+ createStrideMask(I, Factor, NumElts)));
+ return;
+ }
+
+ // Higher factors need more than the two inputs a shuffle has, so gather the
+ // operands into one wide vector, permute it and split the result back up.
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ NumElts * Factor);
+ SmallVector<int, 16> Mask;
+ for (unsigned I = 0; I != Factor; ++I)
+ llvm::append_range(Mask, createStrideMask(I, Factor, NumElts));
+
+ SmallVector<SDValue, 8> InVecs(Node->op_values());
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, InVecs);
+ SDValue Deinterleaved =
+ DAG.getVectorShuffle(WideVT, DL, Concat, DAG.getUNDEF(WideVT), Mask);
+
+ for (unsigned I = 0; I != Factor; ++I)
+ Results.push_back(
+ DAG.getExtractSubvector(DL, VT, Deinterleaved, I * NumElts));
+}
+
void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
switch (Node->getOpcode()) {
case ISD::LOAD: {
@@ -982,6 +1066,12 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
Results.push_back(Node->getOperand(i));
return;
+ case ISD::VECTOR_INTERLEAVE:
+ ExpandVectorInterleaveToShuffles(Node, Results);
+ return;
+ case ISD::VECTOR_DEINTERLEAVE:
+ ExpandVectorDeinterleaveToShuffles(Node, Results);
+ return;
case ISD::SIGN_EXTEND_INREG:
if (SDValue Expanded = ExpandSEXTINREG(Node)) {
Results.push_back(Expanded);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a2b9a14ed9543..2a3be7f9775d1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -13061,18 +13061,6 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
DAG.getVectorIdxConstant(OutNumElts * i, DL));
}
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
- SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(0, 2, OutNumElts));
- SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(1, 2, OutNumElts));
- SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
- setValue(&I, Res);
- return;
- }
-
SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
DAG.getVTList(ValueVTs), SubVecs);
setValue(&I, Res);
@@ -13092,16 +13080,6 @@ void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
"Expected VTs to be the same");
}
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
- unsigned NumElts = InVT.getVectorMinNumElements();
- SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);
- setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
- createInterleaveMask(NumElts, 2)));
- return;
- }
-
SmallVector<EVT, 8> ValueVTs(Factor, InVT);
SDValue Res =
DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 66a6261d2a991..5cf25a44d8ac2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -231,6 +231,9 @@ class AArch64TargetLowering : public TargetLowering {
bool hasPairedLoad(EVT LoadedType, Align &RequiredAlignment) const override;
unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return VT.isScalableVector() || Factor == 3 || Factor % 2 == 0;
+ }
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
ArrayRef<ShuffleVectorInst *> Shuffles,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 972cc256a3386..c30966eaf1843 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -438,6 +438,9 @@ class RISCVTargetLowering : public TargetLowering {
bool isLegalFirstFaultLoad(EVT DataType, Align Alignment) const;
unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return Factor > 2 && Factor <= 8;
+ }
bool fallBackToDAGISel(const Instruction &Inst) const override;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
index 6b49ba0a65340..d367f39445799 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
@@ -157,6 +157,12 @@ define void @f_undef_intrinsic(<8 x i64> %a, ptr %dst) {
;
; CHECK-IADISABLED-LABEL: f_undef_intrinsic:
; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #96]
; CHECK-IADISABLED-NEXT: ret
BB:
%S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> poison, <8 x i64> poison)
@@ -179,6 +185,12 @@ define void @f_poison_intrinsic(<8 x i64> %a, ptr %dst) {
;
; CHECK-IADISABLED-LABEL: f_poison_intrinsic:
; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x0, #96]
; CHECK-IADISABLED-NEXT: ret
BB:
%S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> poison, <8 x i64> poison)
@@ -202,8 +214,14 @@ define void @f_undef_15_intrinsic(<8 x i64> %a, ptr %dst) {
;
; CHECK-IADISABLED-LABEL: f_undef_15_intrinsic:
; CHECK-IADISABLED: // %bb.0: // %BB
-; CHECK-IADISABLED-NEXT: dup v1.2d, v0.d[1]
-; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: zip2 v1.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip2 v3.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #96]
; CHECK-IADISABLED-NEXT: ret
BB:
%a.0 = extractelement <8 x i64> %a, i64 0
@@ -236,17 +254,17 @@ define void @f_undef_1_intrinsic(<8 x i64> %a, ptr %dst) {
;
; CHECK-IADISABLED-LABEL: f_undef_1_intrinsic:
; CHECK-IADISABLED: // %bb.0: // %BB
-; CHECK-IADISABLED-NEXT: dup v4.2d, v3.d[1]
-; CHECK-IADISABLED-NEXT: dup v3.2d, v3.d[0]
-; CHECK-IADISABLED-NEXT: dup v5.2d, v2.d[1]
-; CHECK-IADISABLED-NEXT: dup v2.2d, v2.d[0]
+; CHECK-IADISABLED-NEXT: zip2 v4.2d, v3.2d, v3.2d
+; CHECK-IADISABLED-NEXT: zip1 v3.2d, v3.2d, v3.2d
+; CHECK-IADISABLED-NEXT: zip2 v5.2d, v2.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip2 v6.2d, v1.2d, v1.2d
+; CHECK-IADISABLED-NEXT: zip1 v1.2d, v1.2d, v1.2d
; CHECK-IADISABLED-NEXT: stp q3, q4, [x0, #96]
-; CHECK-IADISABLED-NEXT: dup v4.2d, v1.d[1]
-; CHECK-IADISABLED-NEXT: dup v1.2d, v1.d[0]
-; CHECK-IADISABLED-NEXT: dup v3.2d, v0.d[1]
-; CHECK-IADISABLED-NEXT: dup v0.2d, v0.d[0]
+; CHECK-IADISABLED-NEXT: zip2 v3.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v0.2d
+; CHECK-IADISABLED-NEXT: stp q1, q6, [x0, #32]
; CHECK-IADISABLED-NEXT: stp q2, q5, [x0, #64]
-; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
; CHECK-IADISABLED-NEXT: ret
BB:
diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
index f83739ef6f44f..c56b492bb33cd 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
@@ -896,31 +896,17 @@ define void @aarch64_ilc_idx_ld2_intrinsic(ptr %ptr, i64 %idx) {
; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
; CHECK-NEXT: ret void
;
-; AS-IAENABLED-LABEL: aarch64_ilc_idx_ld2_intrinsic:
-; AS-IAENABLED: // %bb.0: // %entry
-; AS-IAENABLED-NEXT: lsl x8, x1, #2
-; AS-IAENABLED-NEXT: and x9, x8, #0xfffffffffffffff0
-; AS-IAENABLED-NEXT: add x8, x8, #16
-; AS-IAENABLED-NEXT: add x10, x0, x9
-; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
-; AS-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x10]
-; AS-IAENABLED-NEXT: str q0, [x0, x9]
-; AS-IAENABLED-NEXT: str q1, [x0, x8]
-; AS-IAENABLED-NEXT: ret
-;
-; AS-IADISABLED-LABEL: aarch64_ilc_idx_ld2_intrinsic:
-; AS-IADISABLED: // %bb.0: // %entry
-; AS-IADISABLED-NEXT: lsl x8, x1, #2
-; AS-IADISABLED-NEXT: and x9, x8, #0xfffffffffffffff0
-; AS-IADISABLED-NEXT: add x8, x8, #16
-; AS-IADISABLED-NEXT: add x9, x0, x9
-; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
-; AS-IADISABLED-NEXT: ldp q1, q0, [x9]
-; AS-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; AS-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; AS-IADISABLED-NEXT: str q2, [x9]
-; AS-IADISABLED-NEXT: str q0, [x0, x8]
-; AS-IADISABLED-NEXT: ret
+; AS-LABEL: aarch64_ilc_idx_ld2_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: lsl x8, x1, #2
+; AS-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-NEXT: add x8, x8, #16
+; AS-NEXT: add x10, x0, x9
+; AS-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; AS-NEXT: str q0, [x0, x9]
+; AS-NEXT: str q1, [x0, x8]
+; AS-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
@@ -1009,30 +995,17 @@ define void @aarch64_ilc_i32_idx_intrinsic(ptr %ptr, i32 %idx) {
; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
; CHECK-NEXT: ret void
;
-; AS-IAENABLED-LABEL: aarch64_ilc_i32_idx_intrinsic:
-; AS-IAENABLED: // %bb.0: // %entry
-; AS-IAENABLED-NEXT: lsr w8, w1, #2
-; AS-IAENABLED-NEXT: ubfiz x8, x8, #4, #32
-; AS-IAENABLED-NEXT: add x9, x0, x8
-; AS-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x9]
-; AS-IAENABLED-NEXT: add w9, w1, #4
-; AS-IAENABLED-NEXT: lsr w9, w9, #2
-; AS-IAENABLED-NEXT: str q0, [x0, x8]
-; AS-IAENABLED-NEXT: str q1, [x0, w9, uxtw #4]
-; AS-IAENABLED-NEXT: ret
-;
-; AS-IADISABLED-LABEL: aarch64_ilc_i32_idx_intrinsic:
-; AS-IADISABLED: // %bb.0: // %entry
-; AS-IADISABLED-NEXT: lsr w8, w1, #2
-; AS-IADISABLED-NEXT: add w9, w1, #4
-; AS-IADISABLED-NEXT: lsr w9, w9, #2
-; AS-IADISABLED-NEXT: add x8, x0, w8, uxtw #4
-; AS-IADISABLED-NEXT: ldp q1, q0, [x8]
-; AS-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; AS-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; AS-IADISABLED-NEXT: str q2, [x8]
-; AS-IADISABLED-NEXT: str q0, [x0, w9, uxtw #4]
-; AS-IADISABLED-NEXT: ret
+; AS-LABEL: aarch64_ilc_i32_idx_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: lsr w8, w1, #2
+; AS-NEXT: ubfiz x8, x8, #4, #32
+; AS-NEXT: add x9, x0, x8
+; AS-NEXT: ld2 { v0.4s, v1.4s }, [x9]
+; AS-NEXT: add w9, w1, #4
+; AS-NEXT: lsr w9, w9, #2
+; AS-NEXT: str q0, [x0, x8]
+; AS-NEXT: str q1, [x0, w9, uxtw #4]
+; AS-NEXT: ret
entry:
%idx1 = lshr i32 %idx, 2
%a1 = add i32 %idx, 4
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 3ee9df3165c98..ab4e106c6362e 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -251,25 +251,13 @@ entry:
}
define <4 x float> @twosrc_intrinsic(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
-; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: twosrc_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
@@ -286,25 +274,13 @@ entry:
}
define <4 x float> @twosrc2_intrinsic(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc2_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc2_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
-; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: twosrc2_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
@@ -321,21 +297,12 @@ entry:
}
define <4 x float> @vld2_intrinsic(ptr %pSrc) {
-; CHECK-IAENABLED-LABEL: vld2_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld2_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: vld2_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
index 434fb08c3d4b6..e82c7c2849cab 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
@@ -524,10 +524,10 @@ define <4 x float> @mul_triangle_external_use_intrinsic(<4 x float> %a, <4 x flo
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov d2, v0.d[1]
; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: uzp2 v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: uzp1 v5.2s, v1.2s, v3.2s
+; CHECK-NEXT: uzp1 v0.2s, v0.2s, v2.2s
+; CHECK-NEXT: uzp2 v1.2s, v1.2s, v3.2s
; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
@@ -539,9 +539,7 @@ define <4 x float> @mul_triangle_external_use_intrinsic(<4 x float> %a, <4 x flo
; CHECK-NEXT: fneg v3.2s, v3.2s
; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov v3.d[1], v5.d[0]
-; CHECK-NEXT: rev64 v0.4s, v3.4s
-; CHECK-NEXT: uzp1 v0.4s, v3.4s, v0.4s
+; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
; CHECK-NEXT: ret
entry:
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
@@ -569,64 +567,33 @@ entry:
define <4 x float> @multiple_muls_shuffle_external_intrinsic(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
-; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: mov d5, v0.d[1]
-; CHECK-IAENABLED-NEXT: mov d6, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d4, v2.d[1]
-; CHECK-IAENABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip2 v16.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v1.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v16.2s, v7.2s
-; CHECK-IAENABLED-NEXT: zip1 v7.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fneg v6.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fmul v17.2s, v7.2s, v5.2s
-; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: fmul v1.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmla v17.2s, v6.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IAENABLED-NEXT: str d6, [x0]
-; CHECK-IAENABLED-NEXT: fneg v16.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IAENABLED-NEXT: fmla v16.2s, v6.2s, v7.2s
-; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov d4, v1.d[1]
-; CHECK-IADISABLED-NEXT: mov d5, v0.d[1]
-; CHECK-IADISABLED-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-IADISABLED-NEXT: mov d5, v2.d[1]
-; CHECK-IADISABLED-NEXT: fmul v4.2s, v6.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v7.2s
-; CHECK-IADISABLED-NEXT: zip2 v7.2s, v2.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip1 v5.2s, v2.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v16.2s, v7.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v4.2s, v5.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fneg v6.2s, v16.2s
-; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IADISABLED-NEXT: fmla v4.2s, v1.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v6.2s, v1.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IADISABLED-NEXT: str d1, [x0]
-; CHECK-IADISABLED-NEXT: mov v6.d[1], v4.d[0]
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IADISABLED-NEXT: rev64 v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v6.4s, v4.4s
-; CHECK-IADISABLED-NEXT: str q2, [x1]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: multiple_muls_shuffle_external_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d5, v0.d[1]
+; CHECK-NEXT: mov d6, v1.d[1]
+; CHECK-NEXT: mov d4, v2.d[1]
+; CHECK-NEXT: uzp2 v7.2s, v0.2s, v5.2s
+; CHECK-NEXT: uzp2 v16.2s, v1.2s, v6.2s
+; CHECK-NEXT: uzp1 v1.2s, v1.2s, v6.2s
+; CHECK-NEXT: uzp1 v0.2s, v0.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v7.2s
+; CHECK-NEXT: fmul v6.2s, v16.2s, v7.2s
+; CHECK-NEXT: uzp1 v7.2s, v2.2s, v4.2s
+; CHECK-NEXT: uzp2 v4.2s, v2.2s, v4.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v16.2s
+; CHECK-NEXT: fneg v6.2s, v6.2s
+; CHECK-NEXT: fmla v6.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmul v17.2s, v7.2s, v5.2s
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: fmul v1.2s, v4.2s, v5.2s
+; CHECK-NEXT: fmla v17.2s, v6.2s, v4.2s
+; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
+; CHECK-NEXT: str d6, [x0]
+; CHECK-NEXT: fneg v16.2s, v1.2s
+; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
+; CHECK-NEXT: fmla v16.2s, v6.2s, v7.2s
+; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-NEXT: ret
entry:
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
%strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
@@ -667,66 +634,30 @@ entry:
define <4 x float> @multiple_muls_shuffle_external_with_loads_intrinsic(ptr %ptr_a, ptr %ptr_b, ptr %ptr_c, ptr %ptr_d, ptr %p1, ptr %p2) {
-; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v2.2s, v3.2s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v4.2s, v3.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v2.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
-; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v2.2s
-; CHECK-IAENABLED-NEXT: str d4, [x4]
-; CHECK-IAENABLED-NEXT: ldr q5, [x2]
-; CHECK-IAENABLED-NEXT: mov d7, v5.d[1]
-; CHECK-IAENABLED-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v0.2s
-; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x5]
-; CHECK-IAENABLED-NEXT: ldr q1, [x3]
-; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
-; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-IADISABLED-NEXT: ldr q1, [x1]
-; CHECK-IADISABLED-NEXT: mov d2, v0.d[1]
-; CHECK-IADISABLED-NEXT: mov d3, v1.d[1]
-; CHECK-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-IADISABLED-NEXT: zip2 v5.2s, v1.2s, v3.2s
-; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v3.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v4.2s
-; CHECK-IADISABLED-NEXT: str d2, [x4]
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: ldr q3, [x2]
-; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fneg v5.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v2.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v2.2s, v4.2s
-; CHECK-IADISABLED-NEXT: mov v5.d[1], v1.d[0]
-; CHECK-IADISABLED-NEXT: rev64 v0.4s, v5.4s
-; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v5.4s, v0.4s
-; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IADISABLED-NEXT: str q1, [x5]
-; CHECK-IADISABLED-NEXT: ldr q1, [x3]
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #0
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #90
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-NEXT: ld2 { v2.2s, v3.2s }, [x1]
+; CHECK-NEXT: fmul v4.2s, v3.2s, v1.2s
+; CHECK-NEXT: fmul v5.2s, v2.2s, v1.2s
+; CHECK-NEXT: fneg v4.2s, v4.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: str d4, [x4]
+; CHECK-NEXT: ldr q6, [x2]
+; CHECK-NEXT: uzp1 v0.4s, v6.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v6.4s, v0.4s
+; CHECK-NEXT: fmul v3.2s, v0.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
+; CHECK-NEXT: fmla v3.2s, v4.2s, v1.2s
+; CHECK-NEXT: fneg v2.2s, v5.2s
+; CHECK-NEXT: fmla v2.2s, v4.2s, v0.2s
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st2 { v2.2s, v3.2s }, [x5]
+; CHECK-NEXT: ldr q1, [x3]
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #90
+; CHECK-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
@@ -771,80 +702,39 @@ entry:
define <4 x float> @multiple_muls_mul_external_intrinsic(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
-; CHECK-IAENABLED-LABEL: multiple_muls_mul_external_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: mov d4, v0.d[1]
-; CHECK-IAENABLED-NEXT: mov d5, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
-; CHECK-IAENABLED-NEXT: mov d17, v3.d[1]
-; CHECK-IAENABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip2 v19.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip1 v2.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip2 v16.2s, v3.2s, v17.2s
-; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-IAENABLED-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v16.2s, v19.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v18.2s
-; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v19.2s
-; CHECK-IAENABLED-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-IAENABLED-NEXT: fneg v0.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmla v0.2s, v3.2s, v2.2s
-; CHECK-IAENABLED-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v3.2s, v7.2s, v5.2s
-; CHECK-IAENABLED-NEXT: str d4, [x0]
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v0.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v1.2s, v3.2s
-; CHECK-IAENABLED-NEXT: fmla v1.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: mov v0.d[1], v5.d[0]
-; CHECK-IAENABLED-NEXT: rev64 v3.4s, v0.4s
-; CHECK-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x1]
-; CHECK-IAENABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: multiple_muls_mul_external_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov d4, v0.d[1]
-; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
-; CHECK-IADISABLED-NEXT: mov d6, v2.d[1]
-; CHECK-IADISABLED-NEXT: mov d7, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip1 v16.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip2 v17.2s, v1.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip2 v18.2s, v3.2s, v7.2s
-; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v16.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v0.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v7.2s, v4.2s, v18.2s
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmul v0.2s, v2.2s, v18.2s
-; CHECK-IADISABLED-NEXT: fmla v7.2s, v3.2s, v2.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fneg v0.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v1.2s, v16.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v5.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v3.2s, v2.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
-; CHECK-IADISABLED-NEXT: str d2, [x0]
-; CHECK-IADISABLED-NEXT: fmla v3.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v2.2s
-; CHECK-IADISABLED-NEXT: mov v0.d[1], v7.d[0]
-; CHECK-IADISABLED-NEXT: mov v1.d[1], v3.d[0]
-; CHECK-IADISABLED-NEXT: rev64 v3.4s, v0.4s
-; CHECK-IADISABLED-NEXT: rev64 v4.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
-; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v4.4s
-; CHECK-IADISABLED-NEXT: str q1, [x1]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: multiple_muls_mul_external_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d4, v0.d[1]
+; CHECK-NEXT: mov d5, v1.d[1]
+; CHECK-NEXT: mov d16, v2.d[1]
+; CHECK-NEXT: mov d17, v3.d[1]
+; CHECK-NEXT: uzp2 v6.2s, v0.2s, v4.2s
+; CHECK-NEXT: uzp2 v7.2s, v1.2s, v5.2s
+; CHECK-NEXT: uzp2 v19.2s, v2.2s, v16.2s
+; CHECK-NEXT: uzp1 v2.2s, v2.2s, v16.2s
+; CHECK-NEXT: uzp2 v16.2s, v3.2s, v17.2s
+; CHECK-NEXT: uzp1 v0.2s, v0.2s, v4.2s
+; CHECK-NEXT: uzp1 v1.2s, v1.2s, v5.2s
+; CHECK-NEXT: uzp1 v3.2s, v3.2s, v17.2s
+; CHECK-NEXT: fmul v18.2s, v6.2s, v7.2s
+; CHECK-NEXT: fmul v5.2s, v2.2s, v16.2s
+; CHECK-NEXT: fmul v16.2s, v19.2s, v16.2s
+; CHECK-NEXT: fmul v7.2s, v0.2s, v7.2s
+; CHECK-NEXT: fneg v4.2s, v18.2s
+; CHECK-NEXT: fmla v5.2s, v3.2s, v19.2s
+; CHECK-NEXT: fneg v16.2s, v16.2s
+; CHECK-NEXT: fmla v7.2s, v1.2s, v6.2s
+; CHECK-NEXT: fmla v4.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmla v16.2s, v3.2s, v2.2s
+; CHECK-NEXT: fmul v0.2s, v7.2s, v5.2s
+; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
+; CHECK-NEXT: str d4, [x0]
+; CHECK-NEXT: fmla v2.2s, v16.2s, v7.2s
+; CHECK-NEXT: fneg v1.2s, v0.2s
+; CHECK-NEXT: zip1 v0.4s, v16.4s, v5.4s
+; CHECK-NEXT: fmla v1.2s, v16.2s, v4.2s
+; CHECK-NEXT: st2 { v1.2s, v2.2s }, [x1]
+; CHECK-NEXT: ret
entry:
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
%strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
index 355adec955e4b..ad6f77773d9fc 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
@@ -31,8 +31,8 @@ define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -98,8 +98,8 @@ define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -168,10 +168,10 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #90
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v4.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip1 v0.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip2 v3.2d, v2.2d, v1.2d
-; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: uzp2 v4.2d, v0.2d, v3.2d
+; CHECK-NEXT: uzp1 v0.2d, v0.2d, v3.2d
+; CHECK-NEXT: uzp2 v3.2d, v2.2d, v1.2d
+; CHECK-NEXT: uzp1 v1.2d, v2.2d, v1.2d
; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d
; CHECK-NEXT: fadd v1.2d, v4.2d, v3.2d
; CHECK-NEXT: faddp d0, v0.2d
diff --git a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
index 09b6072e8aee2..e427e05848b82 100644
--- a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
@@ -5,91 +5,18 @@
; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IADISABLED
define void @deinterleave_i8_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_i8_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_i8_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: uzp1 v2.16b, v1.16b, v0.16b
-; NEON-IADISABLED-NEXT: uzp2 v0.16b, v1.16b, v0.16b
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_i8_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_i8_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
-; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
-; SVE-FIXED-NEXT: .cfi_offset b8, -16
-; SVE-FIXED-NEXT: ldp q0, q1, [x0]
-; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
-; SVE-FIXED-NEXT: mov z3.b, z1.b[13]
-; SVE-FIXED-NEXT: mov z4.b, z1.b[11]
-; SVE-FIXED-NEXT: mov z5.b, z1.b[9]
-; SVE-FIXED-NEXT: mov z6.b, z1.b[7]
-; SVE-FIXED-NEXT: mov z7.b, z1.b[5]
-; SVE-FIXED-NEXT: mov z18.b, z0.b[15]
-; SVE-FIXED-NEXT: mov z19.b, z0.b[13]
-; SVE-FIXED-NEXT: mov z20.b, z1.b[14]
-; SVE-FIXED-NEXT: mov z21.b, z1.b[12]
-; SVE-FIXED-NEXT: mov z16.b, z1.b[3]
-; SVE-FIXED-NEXT: mov z17.b, z1.b[1]
-; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
-; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
-; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
-; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
-; SVE-FIXED-NEXT: mov z22.b, z1.b[6]
-; SVE-FIXED-NEXT: mov z23.b, z1.b[4]
-; SVE-FIXED-NEXT: zip1 z19.b, z21.b, z20.b
-; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
-; SVE-FIXED-NEXT: mov z21.b, z1.b[8]
-; SVE-FIXED-NEXT: mov z24.b, z1.b[2]
-; SVE-FIXED-NEXT: mov z25.b, z0.b[14]
-; SVE-FIXED-NEXT: mov z26.b, z0.b[12]
-; SVE-FIXED-NEXT: mov z27.b, z0.b[10]
-; SVE-FIXED-NEXT: mov z28.b, z0.b[8]
-; SVE-FIXED-NEXT: mov z29.b, z0.b[6]
-; SVE-FIXED-NEXT: mov z30.b, z0.b[4]
-; SVE-FIXED-NEXT: mov z31.b, z0.b[2]
-; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
-; SVE-FIXED-NEXT: mov z7.b, z0.b[11]
-; SVE-FIXED-NEXT: mov z16.b, z0.b[9]
-; SVE-FIXED-NEXT: mov z17.b, z0.b[7]
-; SVE-FIXED-NEXT: mov z18.b, z0.b[5]
-; SVE-FIXED-NEXT: mov z8.b, z0.b[3]
-; SVE-FIXED-NEXT: zip1 z20.b, z21.b, z20.b
-; SVE-FIXED-NEXT: mov z21.b, z0.b[1]
-; SVE-FIXED-NEXT: zip1 z22.b, z23.b, z22.b
-; SVE-FIXED-NEXT: zip1 z1.b, z1.b, z24.b
-; SVE-FIXED-NEXT: zip1 z23.b, z26.b, z25.b
-; SVE-FIXED-NEXT: zip1 z24.b, z28.b, z27.b
-; SVE-FIXED-NEXT: zip1 z25.b, z30.b, z29.b
-; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z31.b
-; SVE-FIXED-NEXT: zip1 z7.b, z16.b, z7.b
-; SVE-FIXED-NEXT: zip1 z16.b, z18.b, z17.b
-; SVE-FIXED-NEXT: zip1 z17.b, z21.b, z8.b
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z20.h, z19.h
-; SVE-FIXED-NEXT: zip1 z1.h, z1.h, z22.h
-; SVE-FIXED-NEXT: zip1 z18.h, z24.h, z23.h
-; SVE-FIXED-NEXT: zip1 z4.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z25.h
-; SVE-FIXED-NEXT: zip1 z5.h, z7.h, z6.h
-; SVE-FIXED-NEXT: zip1 z6.h, z17.h, z16.h
-; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
-; SVE-FIXED-NEXT: zip1 z2.s, z4.s, z2.s
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z18.s
-; SVE-FIXED-NEXT: zip1 z3.s, z6.s, z5.s
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
-; SVE-FIXED-NEXT: // fake_use: $q1 $z1
-; SVE-FIXED-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload
+; SVE-FIXED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <32 x i8>, ptr %ptr, align 1
%deinterleave = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %load)
@@ -101,55 +28,18 @@ define void @deinterleave_i8_factor2(ptr %ptr) {
}
define void @deinterleave_i16_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_i16_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_i16_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: uzp1 v2.8h, v1.8h, v0.8h
-; NEON-IADISABLED-NEXT: uzp2 v0.8h, v1.8h, v0.8h
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_i16_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_i16_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q1, q0, [x0]
-; SVE-FIXED-NEXT: mov z2.h, z0.h[7]
-; SVE-FIXED-NEXT: mov z3.h, z0.h[5]
-; SVE-FIXED-NEXT: mov z7.h, z0.h[6]
-; SVE-FIXED-NEXT: mov z16.h, z0.h[4]
-; SVE-FIXED-NEXT: mov z17.h, z0.h[2]
-; SVE-FIXED-NEXT: mov z18.h, z1.h[6]
-; SVE-FIXED-NEXT: mov z19.h, z1.h[4]
-; SVE-FIXED-NEXT: mov z20.h, z1.h[2]
-; SVE-FIXED-NEXT: mov z4.h, z0.h[3]
-; SVE-FIXED-NEXT: mov z5.h, z0.h[1]
-; SVE-FIXED-NEXT: mov z6.h, z1.h[7]
-; SVE-FIXED-NEXT: mov z21.h, z1.h[5]
-; SVE-FIXED-NEXT: mov z22.h, z1.h[3]
-; SVE-FIXED-NEXT: mov z23.h, z1.h[1]
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z16.h, z7.h
-; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z17.h
-; SVE-FIXED-NEXT: zip1 z7.h, z19.h, z18.h
-; SVE-FIXED-NEXT: zip1 z1.h, z1.h, z20.h
-; SVE-FIXED-NEXT: zip1 z4.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z5.h, z21.h, z6.h
-; SVE-FIXED-NEXT: zip1 z6.h, z23.h, z22.h
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z3.s
-; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z7.s
-; SVE-FIXED-NEXT: zip1 z2.s, z4.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z6.s, z5.s
-; SVE-FIXED-NEXT: zip1 z0.d, z1.d, z0.d
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
-; SVE-FIXED-NEXT: // fake_use: $q1 $z1
+; SVE-FIXED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <16 x i16>, ptr %ptr, align 2
%deinterleave = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %load)
@@ -161,39 +51,18 @@ define void @deinterleave_i16_factor2(ptr %ptr) {
}
define void @deinterleave_8xi32_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_8xi32_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_8xi32_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_8xi32_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_8xi32_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q0, q1, [x0]
-; SVE-FIXED-NEXT: mov z3.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z4.s, z0.s[2]
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z5.s, z1.s[1]
-; SVE-FIXED-NEXT: mov z6.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z7.s, z0.s[1]
-; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z4.s
-; SVE-FIXED-NEXT: zip1 z2.s, z5.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z7.s, z6.s
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
-; SVE-FIXED-NEXT: // fake_use: $q1 $z1
+; SVE-FIXED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 4
%deinterleave = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %load)
@@ -205,29 +74,18 @@ define void @deinterleave_8xi32_factor2(ptr %ptr) {
}
define void @deinterleave_i64_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_i64_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_i64_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_i64_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_i64_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q1, q0, [x0]
-; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
-; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
-; SVE-FIXED-NEXT: // fake_use: $q2 $z2
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <4 x i64>, ptr %ptr, align 8
%deinterleave = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %load)
@@ -239,39 +97,18 @@ define void @deinterleave_i64_factor2(ptr %ptr) {
}
define void @deinterleave_float_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_float_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_float_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_float_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_float_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q0, q1, [x0]
-; SVE-FIXED-NEXT: mov z3.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z4.s, z0.s[2]
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z5.s, z1.s[1]
-; SVE-FIXED-NEXT: mov z6.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z7.s, z0.s[1]
-; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z4.s
-; SVE-FIXED-NEXT: zip1 z2.s, z5.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z7.s, z6.s
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
-; SVE-FIXED-NEXT: // fake_use: $q1 $z1
+; SVE-FIXED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <8 x float>, ptr %ptr, align 4
%deinterleave = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %load)
@@ -283,29 +120,18 @@ define void @deinterleave_float_factor2(ptr %ptr) {
}
define void @deinterleave_double_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_double_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_double_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_double_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_double_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q1, q0, [x0]
-; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
-; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
-; SVE-FIXED-NEXT: // fake_use: $q2 $z2
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <4 x double>, ptr %ptr, align 8
%deinterleave = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %load)
@@ -317,29 +143,18 @@ define void @deinterleave_double_factor2(ptr %ptr) {
}
define void @deinterleave_ptr_factor2(ptr %ptr) {
-; NEON-IAENABLED-LABEL: deinterleave_ptr_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: // fake_use: $q0
-; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: deinterleave_ptr_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
-; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
-; NEON-IADISABLED-NEXT: // fake_use: $q2
-; NEON-IADISABLED-NEXT: // fake_use: $q0
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: deinterleave_ptr_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q1 $q0_q1
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_ptr_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ldp q1, q0, [x0]
-; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
-; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
-; SVE-FIXED-NEXT: // fake_use: $q2 $z2
-; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; SVE-FIXED-NEXT: // fake_use: $q0
+; SVE-FIXED-NEXT: // fake_use: $q1 $q0_q1
; SVE-FIXED-NEXT: ret
%load = load <4 x ptr>, ptr %ptr, align 8
%deinterleave = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> %load)
@@ -351,57 +166,19 @@ define void @deinterleave_ptr_factor2(ptr %ptr) {
}
define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
-; NEON-IAENABLED-LABEL: interleave_i8_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i8_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
-; NEON-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i8_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i8_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
-; SVE-FIXED-NEXT: mov z3.b, z0.b[15]
-; SVE-FIXED-NEXT: mov z4.b, z1.b[14]
-; SVE-FIXED-NEXT: mov z5.b, z0.b[14]
-; SVE-FIXED-NEXT: mov z6.b, z1.b[13]
-; SVE-FIXED-NEXT: mov z7.b, z0.b[13]
-; SVE-FIXED-NEXT: mov z16.b, z1.b[12]
-; SVE-FIXED-NEXT: mov z17.b, z0.b[12]
-; SVE-FIXED-NEXT: mov z18.b, z1.b[11]
-; SVE-FIXED-NEXT: mov z19.b, z0.b[11]
-; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
-; SVE-FIXED-NEXT: mov z21.b, z0.b[10]
-; SVE-FIXED-NEXT: mov z22.b, z1.b[9]
-; SVE-FIXED-NEXT: mov z23.b, z0.b[9]
-; SVE-FIXED-NEXT: mov z24.b, z1.b[8]
-; SVE-FIXED-NEXT: mov z25.b, z0.b[8]
-; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
-; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
-; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
-; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
-; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
-; SVE-FIXED-NEXT: zip1 z7.b, z21.b, z20.b
-; SVE-FIXED-NEXT: zip1 z16.b, z23.b, z22.b
-; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z1.b
-; SVE-FIXED-NEXT: zip1 z17.b, z25.b, z24.b
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
-; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.b, vl16
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %l, <16 x i8> %r)
store <32 x i8> %interleave, ptr %ptr, align 1
@@ -409,41 +186,19 @@ define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
}
define void @interleave_i16_factor2(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
-; NEON-IAENABLED-LABEL: interleave_i16_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i16_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; NEON-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i16_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i16_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.h, z1.h[7]
-; SVE-FIXED-NEXT: mov z3.h, z0.h[7]
-; SVE-FIXED-NEXT: mov z4.h, z1.h[6]
-; SVE-FIXED-NEXT: mov z5.h, z0.h[6]
-; SVE-FIXED-NEXT: mov z6.h, z1.h[5]
-; SVE-FIXED-NEXT: mov z7.h, z0.h[5]
-; SVE-FIXED-NEXT: mov z16.h, z1.h[4]
-; SVE-FIXED-NEXT: mov z17.h, z0.h[4]
-; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z1.h
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
-; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.h, vl8
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2h { z0.h, z1.h }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %l, <8 x i16> %r)
store <16 x i16> %interleave, ptr %ptr, align 2
@@ -451,33 +206,19 @@ define void @interleave_i16_factor2(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
}
define void @interleave_i32_factor2(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
-; NEON-IAENABLED-LABEL: interleave_i32_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i32_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i32_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i32_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.s, vl4
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %l, <4 x i32> %r)
store <8 x i32> %interleave, ptr %ptr, align 4
@@ -485,27 +226,19 @@ define void @interleave_i32_factor2(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
}
define void @interleave_i64_factor2(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
-; NEON-IAENABLED-LABEL: interleave_i64_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i64_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i64_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i64_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %l, <2 x i64> %r)
store <4 x i64> %interleave, ptr %ptr, align 8
@@ -513,33 +246,19 @@ define void @interleave_i64_factor2(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
}
define void @interleave_float_factor2(ptr %ptr, <4 x float> %l, <4 x float> %r) {
-; NEON-IAENABLED-LABEL: interleave_float_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_float_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_float_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_float_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.s, vl4
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %l, <4 x float> %r)
store <8 x float> %interleave, ptr %ptr, align 4
@@ -547,27 +266,19 @@ define void @interleave_float_factor2(ptr %ptr, <4 x float> %l, <4 x float> %r)
}
define void @interleave_double_factor2(ptr %ptr, <2 x double> %l, <2 x double> %r) {
-; NEON-IAENABLED-LABEL: interleave_double_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_double_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_double_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_double_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %l, <2 x double> %r)
store <4 x double> %interleave, ptr %ptr, align 4
@@ -575,27 +286,19 @@ define void @interleave_double_factor2(ptr %ptr, <2 x double> %l, <2 x double> %
}
define void @interleave_ptr_factor2(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
-; NEON-IAENABLED-LABEL: interleave_ptr_factor2:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_ptr_factor2:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_ptr_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_ptr_factor2:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %l, <2 x ptr> %r)
store <4 x ptr> %interleave, ptr %ptr, align 4
@@ -770,27 +473,19 @@ define void @interleave_wide_ptr_factor2(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
}
define void @interleave_double_factor2_intrinsic(ptr %ptr, <2 x double> %l, <2 x double> %r) {
-; NEON-IAENABLED-LABEL: interleave_double_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_double_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_double_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_double_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %l, <2 x double> %r)
store <4 x double> %interleave, ptr %ptr, align 4
@@ -798,33 +493,19 @@ define void @interleave_double_factor2_intrinsic(ptr %ptr, <2 x double> %l, <2 x
}
define void @interleave_float_factor2_intrinsic(ptr %ptr, <4 x float> %l, <4 x float> %r) {
-; NEON-IAENABLED-LABEL: interleave_float_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_float_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_float_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_float_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.s, vl4
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %l, <4 x float> %r)
store <8 x float> %interleave, ptr %ptr, align 4
@@ -832,41 +513,19 @@ define void @interleave_float_factor2_intrinsic(ptr %ptr, <4 x float> %l, <4 x f
}
define void @interleave_i16_factor2_intrinsic(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
-; NEON-IAENABLED-LABEL: interleave_i16_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i16_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; NEON-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i16_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i16_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.h, z1.h[7]
-; SVE-FIXED-NEXT: mov z3.h, z0.h[7]
-; SVE-FIXED-NEXT: mov z4.h, z1.h[6]
-; SVE-FIXED-NEXT: mov z5.h, z0.h[6]
-; SVE-FIXED-NEXT: mov z6.h, z1.h[5]
-; SVE-FIXED-NEXT: mov z7.h, z0.h[5]
-; SVE-FIXED-NEXT: mov z16.h, z1.h[4]
-; SVE-FIXED-NEXT: mov z17.h, z0.h[4]
-; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z1.h
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
-; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.h, vl8
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2h { z0.h, z1.h }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %l, <8 x i16> %r)
store <16 x i16> %interleave, ptr %ptr, align 2
@@ -874,33 +533,19 @@ define void @interleave_i16_factor2_intrinsic(ptr %ptr, <8 x i16> %l, <8 x i16>
}
define void @interleave_i32_factor2_intrinsic(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
-; NEON-IAENABLED-LABEL: interleave_i32_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i32_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i32_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i32_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
-; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
-; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
-; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
-; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.s, vl4
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %l, <4 x i32> %r)
store <8 x i32> %interleave, ptr %ptr, align 4
@@ -908,27 +553,19 @@ define void @interleave_i32_factor2_intrinsic(ptr %ptr, <4 x i32> %l, <4 x i32>
}
define void @interleave_i64_factor2_intrinsic(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
-; NEON-IAENABLED-LABEL: interleave_i64_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i64_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i64_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i64_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %l, <2 x i64> %r)
store <4 x i64> %interleave, ptr %ptr, align 8
@@ -936,57 +573,19 @@ define void @interleave_i64_factor2_intrinsic(ptr %ptr, <2 x i64> %l, <2 x i64>
}
define void @interleave_i8_factor2_intrinsic(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
-; NEON-IAENABLED-LABEL: interleave_i8_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_i8_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
-; NEON-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_i8_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i8_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
-; SVE-FIXED-NEXT: mov z3.b, z0.b[15]
-; SVE-FIXED-NEXT: mov z4.b, z1.b[14]
-; SVE-FIXED-NEXT: mov z5.b, z0.b[14]
-; SVE-FIXED-NEXT: mov z6.b, z1.b[13]
-; SVE-FIXED-NEXT: mov z7.b, z0.b[13]
-; SVE-FIXED-NEXT: mov z16.b, z1.b[12]
-; SVE-FIXED-NEXT: mov z17.b, z0.b[12]
-; SVE-FIXED-NEXT: mov z18.b, z1.b[11]
-; SVE-FIXED-NEXT: mov z19.b, z0.b[11]
-; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
-; SVE-FIXED-NEXT: mov z21.b, z0.b[10]
-; SVE-FIXED-NEXT: mov z22.b, z1.b[9]
-; SVE-FIXED-NEXT: mov z23.b, z0.b[9]
-; SVE-FIXED-NEXT: mov z24.b, z1.b[8]
-; SVE-FIXED-NEXT: mov z25.b, z0.b[8]
-; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
-; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
-; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
-; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
-; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
-; SVE-FIXED-NEXT: zip1 z7.b, z21.b, z20.b
-; SVE-FIXED-NEXT: zip1 z16.b, z23.b, z22.b
-; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z1.b
-; SVE-FIXED-NEXT: zip1 z17.b, z25.b, z24.b
-; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
-; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
-; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
-; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
-; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
-; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
-; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
-; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ptrue p0.b, vl16
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %l, <16 x i8> %r)
store <32 x i8> %interleave, ptr %ptr, align 1
@@ -994,27 +593,19 @@ define void @interleave_i8_factor2_intrinsic(ptr %ptr, <16 x i8> %l, <16 x i8> %
}
define void @interleave_ptr_factor2_intrinsic(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
-; NEON-IAENABLED-LABEL: interleave_ptr_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: interleave_ptr_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: interleave_ptr_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_ptr_factor2_intrinsic:
; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
-; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
-; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
-; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ptrue p0.d, vl2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; SVE-FIXED-NEXT: st2d { z0.d, z1.d }, p0, [x0]
; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %l, <2 x ptr> %r)
store <4 x ptr> %interleave, ptr %ptr, align 4
@@ -1086,6 +677,5 @@ define void @interleave_wide_ptr_factor2_intrinsic(ptr %ptr, <8 x ptr> %l, <8 x
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; NEON: {{.*}}
; SVE-FIXED-IADISABLED: {{.*}}
; SVE-FIXED-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
index 1278606f5b63f..3c0dc8d99dc76 100644
--- a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -5,20 +5,15 @@
define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) vscale_range(2,2) {
; CHECK-LABEL: store_factor2_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0
-; CHECK-NEXT: ldr z4, [x8]
-; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
-; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
-; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
-; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
-; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
store <32 x i16> %interleaved.vec, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index 426fb4b472cbe..7b4e622034c81 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -5,21 +5,12 @@
; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) {
-; CHECK-SD-LABEL: vector_deinterleave_v2f16_v4f16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: dup v1.2s, v0.s[1]
-; CHECK-SD-NEXT: zip1 v2.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: trn2 v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: fmov d0, d2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vector_deinterleave_v2f16_v4f16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v2.4h, v0.4h, v0.4h
-; CHECK-GI-NEXT: uzp2 v1.4h, v0.4h, v0.4h
-; CHECK-GI-NEXT: fmov d0, d2
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vector_deinterleave_v2f16_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-NEXT: uzp2 v1.4h, v0.4h, v0.4h
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
%retval = call {<2 x half>, <2 x half>} @llvm.vector.deinterleave2.v4f16(<4 x half> %vec)
ret {<2 x half>, <2 x half>} %retval
}
@@ -51,8 +42,8 @@ define {<2 x float>, <2 x float>} @vector_deinterleave_v2f32_v4f32(<4 x float> %
; CHECK-SD-LABEL: vector_deinterleave_v2f32_v4f32:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: mov d1, v0.d[1]
-; CHECK-SD-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: zip2 v1.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp1 v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp2 v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT: fmov d0, d2
; CHECK-SD-NEXT: ret
;
@@ -79,12 +70,19 @@ ret {<4 x float>, <4 x float>} %retval
}
define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2f64_v4f64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d
-; CHECK-NEXT: mov v0.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-NEXT: ret
%retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
ret {<2 x double>, <2 x double>} %retval
}
@@ -125,12 +123,19 @@ define {<4 x i32>, <4 x i32>} @vector_deinterleave_v4i32_v8i32(<8 x i32> %vec) {
}
define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2i64_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d
-; CHECK-NEXT: mov v0.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-NEXT: ret
%retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
ret {<2 x i64>, <2 x i64>} %retval
}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave-odd-factor.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave-odd-factor.ll
new file mode 100644
index 0000000000000..7efb4df681687
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave-odd-factor.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s
+
+define { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @deinterleave5_v10i16(<10 x i16> %vec) {
+; CHECK-LABEL: deinterleave5_v10i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s1, w1
+; CHECK-NEXT: fmov s4, w4
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: fmov s3, w3
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: add x9, sp, #8
+; CHECK-NEXT: fmov s2, w2
+; CHECK-NEXT: mov v1.s[1], w6
+; CHECK-NEXT: ld1 { v4.h }[2], [x9]
+; CHECK-NEXT: mov v0.s[1], w5
+; CHECK-NEXT: ld1 { v3.h }[2], [x8]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q4
+; CHECK-NEXT: mov v2.s[1], w7
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT: ret
+ %result = call { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @llvm.vector.deinterleave5.v10i16(<10 x i16> %vec)
+ ret { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %result
+}
+
+define <10 x i16> @interleave5_v10i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e) {
+; CHECK-LABEL: interleave5_v10i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: adrp x9, .LCPI1_0
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: uzp1 v16.4s, v0.4s, v2.4s
+; CHECK-NEXT: uzp2 v1.4s, v0.4s, v2.4s
+; CHECK-NEXT: mov d17, v16.d[1]
+; CHECK-NEXT: ext v18.16b, v0.16b, v1.16b, #12
+; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI1_0]
+; CHECK-NEXT: mov d19, v18.d[1]
+; CHECK-NEXT: mov v18.s[0], v4.s[0]
+; CHECK-NEXT: mov v4.s[0], v3.s[1]
+; CHECK-NEXT: xtn v1.4h, v4.4s
+; CHECK-NEXT: tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v0.16b
+; CHECK-NEXT: str s1, [x8, #16]
+; CHECK-NEXT: str q0, [x8]
+; CHECK-NEXT: ret
+ %result = call <10 x i16> @llvm.vector.interleave5.v10i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e)
+ ret <10 x i16> %result
+}
+
+define { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @deinterleave7_v14i16(<14 x i16> %vec) {
+; CHECK-LABEL: deinterleave7_v14i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s3, w3
+; CHECK-NEXT: fmov s2, w2
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: fmov s1, w1
+; CHECK-NEXT: fmov s6, w6
+; CHECK-NEXT: add x9, sp, #8
+; CHECK-NEXT: fmov s5, w5
+; CHECK-NEXT: fmov s4, w4
+; CHECK-NEXT: add x10, sp, #16
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: ld1 { v2.h }[2], [x9]
+; CHECK-NEXT: ld1 { v3.h }[2], [x10]
+; CHECK-NEXT: ld1 { v1.h }[2], [x8]
+; CHECK-NEXT: add x8, sp, #24
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: add x10, sp, #40
+; CHECK-NEXT: ld1 { v4.h }[2], [x8]
+; CHECK-NEXT: ld1 { v5.h }[2], [x9]
+; CHECK-NEXT: mov v0.s[1], w7
+; CHECK-NEXT: ld1 { v6.h }[2], [x10]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q4
+; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q5
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d6 killed $d6 killed $q6
+; CHECK-NEXT: ret
+ %result = call { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @llvm.vector.deinterleave7.v14i16(<14 x i16> %vec)
+ ret { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %result
+}
+
+define <14 x i16> @interleave7_v14i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e, <2 x i16> %f, <2 x i16> %g) {
+; CHECK-LABEL: interleave7_v14i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d6 killed $d6 killed $q4_q5_q6 def $q4_q5_q6
+; CHECK-NEXT: fmov d7, d5
+; CHECK-NEXT: fmov d16, d4
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: adrp x9, .LCPI3_1
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: mov v1.d[1], v16.d[0]
+; CHECK-NEXT: mov v16.d[1], v7.d[0]
+; CHECK-NEXT: uzp1 v17.4s, v0.4s, v2.4s
+; CHECK-NEXT: uzp1 v3.4s, v0.4s, v16.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: mov d18, v17.d[1]
+; CHECK-NEXT: ext v19.16b, v3.16b, v0.16b, #8
+; CHECK-NEXT: ext v0.16b, v1.16b, v1.16b, #12
+; CHECK-NEXT: mov v2.16b, v19.16b
+; CHECK-NEXT: mov v2.s[2], v6.s[0]
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI3_1]
+; CHECK-NEXT: adrp x9, .LCPI3_0
+; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI3_0]
+; CHECK-NEXT: mov d5, v4.d[1]
+; CHECK-NEXT: mov d20, v2.d[1]
+; CHECK-NEXT: mov v6.s[0], v7.s[1]
+; CHECK-NEXT: tbl v1.16b, { v17.16b, v18.16b, v19.16b, v20.16b }, v1.16b
+; CHECK-NEXT: tbl v0.16b, { v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-NEXT: str q1, [x8]
+; CHECK-NEXT: mov s2, v0.s[2]
+; CHECK-NEXT: str d0, [x8, #16]
+; CHECK-NEXT: str s2, [x8, #24]
+; CHECK-NEXT: ret
+ %result = call <14 x i16> @llvm.vector.interleave7.v14i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d, <2 x i16> %e, <2 x i16> %f, <2 x i16> %g)
+ ret <14 x i16> %result
+}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
index ab06111fb52ff..eb6326e946fb2 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
@@ -14,22 +14,12 @@ define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) {
}
define <8 x half> @interleave2_v8f16(<4 x half> %vec0, <4 x half> %vec1) {
-; CHECK-SD-LABEL: interleave2_v8f16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT: adrp x8, .LCPI1_0
-; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-SD-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: interleave2_v8f16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: interleave2_v8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ret
%retval = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %vec0, <4 x half> %vec1)
ret <8 x half> %retval
}
@@ -46,21 +36,12 @@ define <16 x half> @interleave2_v16f16(<8 x half> %vec0, <8 x half> %vec1) {
}
define <4 x float> @interleave2_v4f32(<2 x float> %vec0, <2 x float> %vec1) {
-; CHECK-SD-LABEL: interleave2_v4f32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT: rev64 v1.4s, v0.4s
-; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: interleave2_v4f32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: interleave2_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
%retval = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %vec0, <2 x float> %vec1)
ret <4 x float> %retval
}
@@ -187,11 +168,10 @@ define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) {
; CHECK-SD-LABEL: interleave2_diff_nonconst_splat_v4i16:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w0
-; CHECK-SD-NEXT: mov v0.h[2], w1
+; CHECK-SD-NEXT: mov v0.h[1], w1
+; CHECK-SD-NEXT: mov v0.h[2], w0
; CHECK-SD-NEXT: mov v0.h[3], w1
-; CHECK-SD-NEXT: rev32 v1.4h, v0.4h
-; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: interleave2_diff_nonconst_splat_v4i16:
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
index da917119c6b25..d8ff031239b41 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
@@ -5,25 +5,17 @@
define { <16 x i8>, <16 x i8> } @foo_ld2_v16i8(<16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 v1.16b, v0.16b, v0.16b
; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b
; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: umov w9, v1.h[0]
; CHECK-NEXT: umov w8, v0.h[0]
-; CHECK-NEXT: bfi w8, w9, #16, #16
+; CHECK-NEXT: bfi w8, w8, #16, #16
; CHECK-NEXT: tbz w8, #0, .LBB0_2
; CHECK-NEXT: // %bb.1: // %cond.load
; CHECK-NEXT: ldr b1, [x0]
@@ -349,10 +341,11 @@ define { <8 x i16>, <8 x i16> } @foo_ld2_v8i16(<8 x i1> %mask, ptr %p) {
define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v4f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h
; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: shl v0.8b, v0.8b, #7
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
@@ -419,10 +412,11 @@ define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) {
define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v2f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
-; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
@@ -453,8 +447,8 @@ define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) {
; CHECK-NEXT: add x8, x0, #24
; CHECK-NEXT: ld1 { v2.d }[1], [x8]
; CHECK-NEXT: .LBB3_8: // %else8
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v2.2d
-; CHECK-NEXT: zip2 v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v2.2d
+; CHECK-NEXT: uzp2 v1.2d, v1.2d, v2.2d
; CHECK-NEXT: ret
%interleaved.mask = call <4 x i1> @llvm.vector.interleave2.v4i1(<2 x i1> %mask, <2 x i1> %mask)
%wide.masked.vec = call <4 x double> @llvm.masked.load.v4f64.p0(ptr %p, i32 8, <4 x i1> %interleaved.mask, <4 x double> poison)
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
index 972be0fca2089..5155be8cfed0d 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
@@ -5,26 +5,18 @@
define void @foo_st2_v16i8(<16 x i1> %mask, <16 x i8> %val1, <16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 v3.16b, v0.16b, v0.16b
; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b
; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: shl v3.16b, v3.16b, #7
+; CHECK-NEXT: ldr q3, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: cmlt v3.16b, v3.16b, #0
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-NEXT: and v3.16b, v3.16b, v4.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: umov w9, v3.h[0]
; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: zip1 v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bfi w8, w9, #16, #16
+; CHECK-NEXT: bfi w8, w8, #16, #16
; CHECK-NEXT: tbnz w8, #0, .LBB0_33
; CHECK-NEXT: // %bb.1: // %else
; CHECK-NEXT: tbnz w8, #1, .LBB0_34
@@ -345,10 +337,11 @@ define void @foo_st2_v8i16(<8 x i1> %mask, <8 x i16> %val1, <8 x i16> %val2, ptr
define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h
; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: shl v0.8b, v0.8b, #7
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: and v0.8b, v0.8b, v3.8b
@@ -413,10 +406,11 @@ define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr
define void @foo_st2_v2i64(<2 x i1> %mask, <2 x i64> %val1, <2 x i64> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI3_0]
-; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
; CHECK-NEXT: and v0.8b, v0.8b, v3.8b
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index bedce9ec6a86e..5f75d41378361 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -2195,23 +2195,12 @@ define <4 x i1> @load_large_vector_intrinsic(ptr %p) {
}
define void @store_factor2_intrinsic(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
-; NEON-IAENABLED-LABEL: store_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; NEON-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; NEON-IAENABLED-NEXT: st2 { v0.8b, v1.8b }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; NEON-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; NEON-IADISABLED-NEXT: adrp x8, .LCPI41_0
-; NEON-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; NEON-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI41_0]
-; NEON-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; NEON-IADISABLED-NEXT: str q0, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; NEON-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; NEON-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_factor2_intrinsic:
; NO_NEON: // %bb.0:
@@ -2873,19 +2862,12 @@ define void @store_general_mask_factor4_undefmulti_intrinsic(ptr %ptr, <32 x i32
}
define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
-; NEON-IAENABLED-LABEL: store_ptrvec_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_ptrvec_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_ptrvec_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_ptrvec_factor2_intrinsic:
; NO_NEON: // %bb.0:
@@ -2942,19 +2924,12 @@ define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %
}
define void @store_undef_mask_factor2_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
-; NEON-IAENABLED-LABEL: store_undef_mask_factor2_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_undef_mask_factor2_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_undef_mask_factor2_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_undef_mask_factor2_intrinsic:
; NO_NEON: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll b/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
index 951f356ee1da5..32f43f03b2b16 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
@@ -2425,39 +2425,19 @@ entry:
}
define void @test_ldnp_interleaved_load2_v2i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: str d0, [x1]
+; CHECK-LE-NEXT: str d1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <4 x i32>, ptr %ptr, align 4, !nontemporal !0
%deinterleaved = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %loaded)
@@ -2470,37 +2450,19 @@ entry:
define void @test_ldnp_interleaved_load2_v4i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: xtn v1.4h, v0.4s
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
-; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
-; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-NEXT: str d0, [x1]
+; CHECK-LE-NEXT: str d1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x i16>, ptr %ptr, align 2, !nontemporal !0
%deinterleaved = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> %loaded)
@@ -2513,37 +2475,19 @@ entry:
define void @test_ldnp_interleaved_load2_v8i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: xtn v1.8b, v0.8h
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
-; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
-; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.8b }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.8b }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-LE-NEXT: str d0, [x1]
+; CHECK-LE-NEXT: str d1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v1.16b, v0.16b, v0.16b
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-BE-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-NEXT: st1 { v1.8b }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x i8>, ptr %ptr, align 1, !nontemporal !0
%deinterleaved = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> %loaded)
@@ -2556,39 +2500,19 @@ entry:
define void @test_ldnp_interleaved_load2_v2f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: str d0, [x1]
+; CHECK-LE-NEXT: str d1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <4 x float>, ptr %ptr, align 4, !nontemporal !0
%deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %loaded)
@@ -2601,37 +2525,19 @@ entry:
define void @test_ldnp_interleaved_load2_v4f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
-; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
-; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-NEXT: str d0, [x1]
+; CHECK-LE-NEXT: str d1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x half>, ptr %ptr, align 2, !nontemporal !0
%deinterleaved = call { <4 x half>, <4 x half> } @llvm.vector.deinterleave2.v8f16(<8 x half> %loaded)
@@ -2644,39 +2550,19 @@ entry:
define void @test_ldnp_interleaved_load2_v2i64_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <4 x i64>, ptr %ptr, align 8, !nontemporal !0
%deinterleaved = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %loaded)
@@ -2689,39 +2575,19 @@ entry:
define void @test_ldnp_interleaved_load2_v4i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x i32>, ptr %ptr, align 4, !nontemporal !0
%deinterleaved = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %loaded)
@@ -2734,39 +2600,19 @@ entry:
define void @test_ldnp_interleaved_load2_v8i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x i16>, ptr %ptr, align 2, !nontemporal !0
%deinterleaved = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %loaded)
@@ -2779,39 +2625,19 @@ entry:
define void @test_ldnp_interleaved_load2_v16i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.16b }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.16b }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-BE-NEXT: st1 { v0.16b }, [x1]
+; CHECK-BE-NEXT: st1 { v1.16b }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <32 x i8>, ptr %ptr, align 1, !nontemporal !0
%deinterleaved = call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %loaded)
@@ -2824,39 +2650,19 @@ entry:
define void @test_ldnp_interleaved_load2_v2f64_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <4 x double>, ptr %ptr, align 8, !nontemporal !0
%deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %loaded)
@@ -2869,39 +2675,19 @@ entry:
define void @test_ldnp_interleaved_load2_v4f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x float>, ptr %ptr, align 4, !nontemporal !0
%deinterleaved = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %loaded)
@@ -2914,39 +2700,19 @@ entry:
define void @test_ldnp_interleaved_load2_v8f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
-; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
-; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
-; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
-; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-NEXT: str q0, [x1]
+; CHECK-LE-NEXT: str q1, [x2]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x half>, ptr %ptr, align 2, !nontemporal !0
%deinterleaved = call { <8 x half>, <8 x half> } @llvm.vector.deinterleave2.v16f16(<16 x half> %loaded)
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
index ca1f86d83710c..452e2e56c8625 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
@@ -166,39 +166,19 @@ entry:
}
define void @test_stnp_interleaved_store2_v2i32_Os_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #0 {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
-;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -207,39 +187,19 @@ entry:
define void @test_stnp_interleaved_store2_v2i32_Oz_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #1 {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
-;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -248,37 +208,19 @@ entry:
define void @test_stnp_interleaved_store2_v2i64_Os_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #0 {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: stp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
-;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
@@ -287,37 +229,19 @@ entry:
define void @test_stnp_interleaved_store2_v2i64_Oz_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #1 {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: stp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
-;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
@@ -328,6 +252,3 @@ entry:
attributes #0 = { optsize }
attributes #1 = { minsize optsize }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-BE: {{.*}}
-; CHECK-LE: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
index 76b77530f2d16..9cbc98fc96814 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
@@ -2376,40 +2376,19 @@ entry:
}
define void @test_stnp_interleaved_store2_v2i32_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -2418,44 +2397,19 @@ entry:
define void @test_stnp_interleaved_store2_v4i16_intrinsic(<4 x i16> %v0, <4 x i16> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI39_0
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI39_0]
-; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
-; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI39_0
-; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI39_0
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <8 x i16> @llvm.vector.interleave2.v8i16(<4 x i16> %v0, <4 x i16> %v1)
store <8 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
@@ -2464,43 +2418,19 @@ entry:
define void @test_stnp_interleaved_store2_v8i8_intrinsic(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.8b, v1.8b }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI40_0
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI40_0]
-; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.8b, v1.8b
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.8b, v0.8b
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.8b, v2.8b }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.8b, v0.8b
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.8b, v1.8b
-; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI40_0
-; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI40_0
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.8b, v1.8b
+; CHECK-BE-NEXT: rev64 v1.8b, v0.8b
+; CHECK-BE-NEXT: st2 { v1.8b, v2.8b }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> %v0, <8 x i8> %v1)
store <16 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
@@ -2509,40 +2439,19 @@ entry:
define void @test_stnp_interleaved_store2_v2f32_intrinsic(<2 x float> %v0, <2 x float> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %v0, <2 x float> %v1)
store <4 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -2551,44 +2460,19 @@ entry:
define void @test_stnp_interleaved_store2_v4f16_intrinsic(<4 x half> %v0, <4 x half> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI42_0
-; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI42_0]
-; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
-; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-NEXT: st2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
-; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI42_0
-; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI42_0
-; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %v0, <4 x half> %v1)
store <8 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
@@ -2597,37 +2481,19 @@ entry:
define void @test_stnp_interleaved_store2_v2i64_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
@@ -2636,41 +2502,21 @@ entry:
define void @test_stnp_interleaved_store2_v4i32_intrinsic(<4 x i32> %v0, <4 x i32> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
store <8 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -2679,41 +2525,21 @@ entry:
define void @test_stnp_interleaved_store2_v8i16_intrinsic(<8 x i16> %v0, <8 x i16> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %v0, <8 x i16> %v1)
store <16 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
@@ -2722,41 +2548,21 @@ entry:
define void @test_stnp_interleaved_store2_v16i8_intrinsic(<16 x i8> %v0, <16 x i8> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.16b, v1.16b
-; CHECK-BE-IAENABLED-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.16b, v2.16b }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.16b, v2.16b }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %v0, <16 x i8> %v1)
store <32 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
@@ -2765,37 +2571,19 @@ entry:
define void @test_stnp_interleaved_store2_v2f64_intrinsic(<2 x double> %v0, <2 x double> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %v0, <2 x double> %v1)
store <4 x double> %interleave, ptr %ptr, align 8, !nontemporal !0
@@ -2804,41 +2592,21 @@ entry:
define void @test_stnp_interleaved_store2_v4f32_intrinsic(<4 x float> %v0, <4 x float> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %v0, <4 x float> %v1)
store <8 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
@@ -2847,41 +2615,21 @@ entry:
define void @test_stnp_interleaved_store2_v8f16_intrinsic(<8 x half> %v0, <8 x half> %v1, ptr %ptr) {
-; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
-; CHECK-LE-IAENABLED: // %bb.0: // %entry
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-IAENABLED-NEXT: ret
-;
-; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
-; CHECK-LE-IADISABLED: // %bb.0: // %entry
-; CHECK-LE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-IADISABLED-NEXT: ret
-;
-; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
-; CHECK-BE-IAENABLED: // %bb.0: // %entry
-; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
-; CHECK-BE-IAENABLED-NEXT: ret
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-NEXT: ret
;
-; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
-; CHECK-BE-IADISABLED: // %bb.0: // %entry
-; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
-; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
-; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
-; CHECK-BE-IADISABLED-NEXT: ret
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-NEXT: ret
entry:
%interleave = call <16 x half> @llvm.vector.interleave2.v16f16(<8 x half> %v0, <8 x half> %v1)
store <16 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
@@ -3503,11 +3251,14 @@ entry:
define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic(<3 x i32> %v0, <3 x i32> %v1, ptr %ptr) {
; CHECK-LE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: zip1 v2.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: zip2 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: mov d3, v2.d[1]
-; CHECK-LE-NEXT: str d0, [x0, #16]
-; CHECK-LE-NEXT: stnp d2, d3, [x0]
+; CHECK-LE-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ext v1.16b, v0.16b, v2.16b, #12
+; CHECK-LE-NEXT: mov v0.s[3], v1.s[0]
+; CHECK-LE-NEXT: mov d2, v0.d[1]
+; CHECK-LE-NEXT: ext v1.16b, v1.16b, v0.16b, #4
+; CHECK-LE-NEXT: stnp d0, d2, [x0]
+; CHECK-LE-NEXT: str d1, [x0, #16]
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic:
@@ -3518,6 +3269,9 @@ define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic(<3
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: zip2 v2.4s, v0.4s, v1.4s
; CHECK-BE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ext v1.16b, v0.16b, v2.16b, #12
+; CHECK-BE-NEXT: ext v2.16b, v1.16b, v0.16b, #4
+; CHECK-BE-NEXT: mov v0.s[3], v1.s[0]
; CHECK-BE-NEXT: rev64 v1.4s, v2.4s
; CHECK-BE-NEXT: st1 { v0.4s }, [x0]
; CHECK-BE-NEXT: str d1, [x0, #16]
@@ -3532,54 +3286,48 @@ entry:
define void @test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic(<2 x i24> %v0, <2 x i24> %v1, ptr %ptr) {
; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-NEXT: mov v2.16b, v0.16b
-; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-LE-NEXT: fmov w9, s0
+; CHECK-LE-NEXT: zip2 v2.2s, v0.2s, v1.2s
+; CHECK-LE-NEXT: zip1 v0.2s, v0.2s, v1.2s
+; CHECK-LE-NEXT: fmov w9, s2
+; CHECK-LE-NEXT: mov w8, v2.s[1]
+; CHECK-LE-NEXT: mov w10, v0.s[1]
+; CHECK-LE-NEXT: str h2, [x0, #6]
; CHECK-LE-NEXT: str h0, [x0]
-; CHECK-LE-NEXT: mov w8, v1.s[1]
-; CHECK-LE-NEXT: mov v2.d[1], v1.d[0]
; CHECK-LE-NEXT: lsr w9, w9, #16
-; CHECK-LE-NEXT: strb w9, [x0, #2]
-; CHECK-LE-NEXT: lsr w9, w8, #16
; CHECK-LE-NEXT: sturh w8, [x0, #9]
-; CHECK-LE-NEXT: rev64 v3.4s, v2.4s
-; CHECK-LE-NEXT: strb w9, [x0, #11]
-; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-LE-NEXT: mov w10, v2.s[2]
-; CHECK-LE-NEXT: mov w11, v2.s[1]
-; CHECK-LE-NEXT: lsr w8, w10, #16
-; CHECK-LE-NEXT: lsr w9, w11, #16
-; CHECK-LE-NEXT: strh w10, [x0, #6]
-; CHECK-LE-NEXT: sturh w11, [x0, #3]
-; CHECK-LE-NEXT: strb w8, [x0, #8]
-; CHECK-LE-NEXT: strb w9, [x0, #5]
+; CHECK-LE-NEXT: lsr w8, w8, #16
+; CHECK-LE-NEXT: strb w9, [x0, #8]
+; CHECK-LE-NEXT: fmov w9, s0
+; CHECK-LE-NEXT: sturh w10, [x0, #3]
+; CHECK-LE-NEXT: lsr w10, w10, #16
+; CHECK-LE-NEXT: strb w8, [x0, #11]
+; CHECK-LE-NEXT: lsr w8, w9, #16
+; CHECK-LE-NEXT: strb w10, [x0, #5]
+; CHECK-LE-NEXT: strb w8, [x0, #2]
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic:
; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
; CHECK-BE-NEXT: rev64 v1.2s, v1.2s
-; CHECK-BE-NEXT: mov v2.16b, v0.16b
-; CHECK-BE-NEXT: fmov w9, s0
-; CHECK-BE-NEXT: mov w8, v1.s[1]
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: zip2 v2.2s, v0.2s, v1.2s
+; CHECK-BE-NEXT: zip1 v0.2s, v0.2s, v1.2s
+; CHECK-BE-NEXT: fmov w9, s2
+; CHECK-BE-NEXT: mov w8, v2.s[1]
+; CHECK-BE-NEXT: mov w10, v0.s[1]
+; CHECK-BE-NEXT: stur b2, [x0, #8]
; CHECK-BE-NEXT: stur b0, [x0, #2]
-; CHECK-BE-NEXT: mov v2.d[1], v1.d[0]
; CHECK-BE-NEXT: lsr w9, w9, #8
; CHECK-BE-NEXT: strb w8, [x0, #11]
-; CHECK-BE-NEXT: strh w9, [x0]
-; CHECK-BE-NEXT: lsr w9, w8, #8
-; CHECK-BE-NEXT: rev64 v3.4s, v2.4s
-; CHECK-BE-NEXT: sturh w9, [x0, #9]
-; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-BE-NEXT: mov w10, v2.s[2]
-; CHECK-BE-NEXT: mov w11, v2.s[1]
-; CHECK-BE-NEXT: lsr w8, w10, #8
-; CHECK-BE-NEXT: lsr w9, w11, #8
-; CHECK-BE-NEXT: strb w10, [x0, #8]
-; CHECK-BE-NEXT: strb w11, [x0, #5]
-; CHECK-BE-NEXT: strh w8, [x0, #6]
-; CHECK-BE-NEXT: sturh w9, [x0, #3]
+; CHECK-BE-NEXT: lsr w8, w8, #8
+; CHECK-BE-NEXT: strh w9, [x0, #6]
+; CHECK-BE-NEXT: fmov w9, s0
+; CHECK-BE-NEXT: strb w10, [x0, #5]
+; CHECK-BE-NEXT: lsr w10, w10, #8
+; CHECK-BE-NEXT: sturh w8, [x0, #9]
+; CHECK-BE-NEXT: lsr w8, w9, #8
+; CHECK-BE-NEXT: sturh w10, [x0, #3]
+; CHECK-BE-NEXT: strh w8, [x0]
; CHECK-BE-NEXT: ret
entry:
%interleave = call <4 x i24> @llvm.vector.interleave2.v4i24(<2 x i24> %v0, <2 x i24> %v1)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index 4300bcf317d12..1a51a03a4f5b3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -50,25 +50,13 @@ exit:
define void @hang_when_merging_stores_after_legalisation_intrinsic(ptr %a, <2 x i32> %b) vscale_range(2,2) #0 {
-; CHECK-IAENABLED-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-IAENABLED-NEXT: dup v0.4s, v0.s[0]
-; CHECK-IAENABLED-NEXT: mov v1.16b, v0.16b
-; CHECK-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-IADISABLED-NEXT: adrp x8, .LCPI2_0
-; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI2_0
-; CHECK-IADISABLED-NEXT: dup v0.4s, v0.s[0]
-; CHECK-IADISABLED-NEXT: ldr z1, [x8]
-; CHECK-IADISABLED-NEXT: mov z0.q, q0
-; CHECK-IADISABLED-NEXT: tbl z0.s, { z0.s }, z1.s
-; CHECK-IADISABLED-NEXT: str z0, [x0]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: dup v0.4s, v0.s[0]
+; CHECK-NEXT: mov v1.16b, v0.16b
+; CHECK-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-NEXT: ret
%splat = shufflevector <2 x i32> %b, <2 x i32> poison, <4 x i32> zeroinitializer
%interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %splat, <4 x i32> %splat)
store <8 x i32> %interleaved.vec, ptr %a, align 4
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index 30f944e0a050d..db042ba1d2bc2 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1654,20 +1654,15 @@ define void @deinterleave_nxptr_factor2(ptr %ptr) #2 {
define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
; CHECK-LABEL: store_factor2_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: adrp x8, .LCPI29_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI29_0
-; CHECK-NEXT: ldr z4, [x8]
-; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
-; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
-; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
-; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
-; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
store <32 x i16> %interleaved.vec, ptr %ptr, align 4
@@ -1722,20 +1717,15 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4
define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
; CHECK-LABEL: store_ptrvec_factor2_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: adrp x8, .LCPI32_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI32_0
-; CHECK-NEXT: ldr z4, [x8]
-; CHECK-NEXT: splice z1.d, p0, z1.d, z3.d
-; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
-; CHECK-NEXT: tbl z1.d, { z1.d }, z4.d
-; CHECK-NEXT: tbl z0.d, { z0.d }, z4.d
-; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <8 x ptr> @llvm.vector.interleave2.v8p0(<4 x ptr> %v0, <4 x ptr> %v1)
store <8 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -1823,22 +1813,15 @@ define void @store_factor2_wide_intrinsic(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1
define void @store_min_not_max_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
; CHECK-LABEL: store_min_not_max_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: adrp x8, .LCPI36_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI36_0
-; CHECK-NEXT: ptrue p1.d, vl4
-; CHECK-NEXT: splice z1.d, p0, z1.d, z3.d
-; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x8]
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: tbl z1.d, { z1.d }, z2.d
-; CHECK-NEXT: tbl z0.d, { z0.d }, z2.d
-; CHECK-NEXT: st1d { z1.d }, p1, [x0, x8, lsl #3]
-; CHECK-NEXT: st1d { z0.d }, p1, [x0]
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> %v0, <4 x i64> %v1)
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
@@ -1848,20 +1831,15 @@ define void @store_min_not_max_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1)
define void @store_min_ge_type_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
; CHECK-LABEL: store_min_ge_type_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: adrp x8, .LCPI37_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI37_0
-; CHECK-NEXT: splice z2.d, p0, z2.d, z3.d
-; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ldr z1, [x8]
-; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
-; CHECK-NEXT: tbl z0.d, { z0.d }, z1.d
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> %v0, <4 x i64> %v1)
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
@@ -1916,20 +1894,15 @@ define void @store_float_factor3_intrinsic(ptr %ptr, <8 x float> %v0, <8 x float
define void @store_half_factor2_intrinsic(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
; CHECK-LABEL: store_half_factor2_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: adrp x8, .LCPI40_0
-; CHECK-NEXT: add x8, x8, :lo12:.LCPI40_0
-; CHECK-NEXT: ldr z4, [x8]
-; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
-; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
-; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
-; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
-; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <32 x half> @llvm.vector.interleave2.v32f16(<16 x half> %v0, <16 x half> %v1)
store <32 x half> %interleaved.vec, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
index d87b6b0404874..4f4dac8fe9f65 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
@@ -9,23 +9,15 @@ define void @store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1) {
; CHECK-COMPAT-LABEL: store_factor2_64bit:
; CHECK-COMPAT: // %bb.0:
; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
-; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
-; CHECK-COMPAT-NEXT: adrp x8, .LCPI0_0
-; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-COMPAT-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z1.s
-; CHECK-COMPAT-NEXT: str q0, [x0]
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-COMPAT-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-COMPAT-NEXT: ret
;
; CHECK-STREAMING-LABEL: store_factor2_64bit:
; CHECK-STREAMING: // %bb.0:
; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
-; CHECK-STREAMING-NEXT: adrp x8, .LCPI0_0
-; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
-; CHECK-STREAMING-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z1.s
-; CHECK-STREAMING-NEXT: str q0, [x0]
+; CHECK-STREAMING-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-STREAMING-NEXT: ret
%interleaved = call <4 x i32> @llvm.vector.interleave2.v4i32(
<2 x i32> %v0, <2 x i32> %v1)
@@ -36,30 +28,16 @@ define void @store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1) {
define void @store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
; CHECK-COMPAT-LABEL: store_factor2_128bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-COMPAT-NEXT: mov z2.s, z1.s[3]
-; CHECK-COMPAT-NEXT: mov z3.s, z0.s[3]
-; CHECK-COMPAT-NEXT: mov z4.s, z1.s[2]
-; CHECK-COMPAT-NEXT: mov z5.s, z0.s[2]
-; CHECK-COMPAT-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-COMPAT-NEXT: zip1 z2.s, z3.s, z2.s
-; CHECK-COMPAT-NEXT: zip1 z3.s, z5.s, z4.s
-; CHECK-COMPAT-NEXT: zip1 z1.d, z3.d, z2.d
-; CHECK-COMPAT-NEXT: stp q0, q1, [x0]
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-COMPAT-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-COMPAT-NEXT: ret
;
; CHECK-STREAMING-LABEL: store_factor2_128bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: mov z2.s, z1.s[3]
-; CHECK-STREAMING-NEXT: mov z3.s, z0.s[3]
-; CHECK-STREAMING-NEXT: mov z4.s, z1.s[2]
-; CHECK-STREAMING-NEXT: mov z5.s, z0.s[2]
-; CHECK-STREAMING-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-STREAMING-NEXT: zip1 z2.s, z3.s, z2.s
-; CHECK-STREAMING-NEXT: zip1 z3.s, z5.s, z4.s
-; CHECK-STREAMING-NEXT: zip1 z1.d, z3.d, z2.d
-; CHECK-STREAMING-NEXT: stp q0, q1, [x0]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-STREAMING-NEXT: ret
%interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
<4 x i32> %v0, <4 x i32> %v1)
@@ -157,40 +135,22 @@ define void @masked_store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1,
; CHECK-COMPAT-LABEL: masked_store_factor2_64bit:
; CHECK-COMPAT: // %bb.0:
; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 def $z2
-; CHECK-COMPAT-NEXT: mov z3.s, z2.s[1]
; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
-; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
-; CHECK-COMPAT-NEXT: adrp x8, .LCPI6_0
-; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
-; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z3.h
-; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z2.h
-; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1 def $z0_z1
; CHECK-COMPAT-NEXT: lsl z2.s, z2.s, #31
-; CHECK-COMPAT-NEXT: asr z1.s, z2.s, #31
-; CHECK-COMPAT-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
-; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z2.s
-; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-COMPAT-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1 def $z0_z1
+; CHECK-COMPAT-NEXT: asr z2.s, z2.s, #31
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; CHECK-COMPAT-NEXT: st2w { z0.s, z1.s }, p1, [x0]
; CHECK-COMPAT-NEXT: ret
;
; CHECK-STREAMING-LABEL: masked_store_factor2_64bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: mov z3.s, z2.s[1]
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
-; CHECK-STREAMING-NEXT: adrp x8, .LCPI6_0
-; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
-; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z3.h
-; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z2.h
-; CHECK-STREAMING-NEXT: uunpklo z2.s, z2.h
; CHECK-STREAMING-NEXT: lsl z2.s, z2.s, #31
-; CHECK-STREAMING-NEXT: asr z1.s, z2.s, #31
-; CHECK-STREAMING-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
-; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z2.s
-; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-STREAMING-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
+; CHECK-STREAMING-NEXT: asr z2.s, z2.s, #31
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; CHECK-STREAMING-NEXT: st2w { z0.s, z1.s }, p1, [x0]
; CHECK-STREAMING-NEXT: ret
%interleaved = call <4 x i32> @llvm.vector.interleave2.v4i32(
<2 x i32> %v0, <2 x i32> %v1)
@@ -204,61 +164,25 @@ define void @masked_store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1,
define void @masked_store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i1> %mask) {
; CHECK-COMPAT-LABEL: masked_store_factor2_128bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: adrp x8, .LCPI7_0
; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 def $z2
-; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-COMPAT-NEXT: mov z4.s, z1.s[3]
-; CHECK-COMPAT-NEXT: mov z5.s, z0.s[3]
-; CHECK-COMPAT-NEXT: ldr q3, [x8, :lo12:.LCPI7_0]
-; CHECK-COMPAT-NEXT: mov z6.s, z1.s[2]
-; CHECK-COMPAT-NEXT: mov z7.s, z0.s[2]
; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
-; CHECK-COMPAT-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-COMPAT-NEXT: mov x8, #4 // =0x4
-; CHECK-COMPAT-NEXT: tbl z3.h, { z2.h }, z3.h
-; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z2.h
-; CHECK-COMPAT-NEXT: zip1 z4.s, z5.s, z4.s
-; CHECK-COMPAT-NEXT: zip1 z5.s, z7.s, z6.s
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
-; CHECK-COMPAT-NEXT: uunpklo z3.s, z3.h
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-COMPAT-NEXT: lsl z2.s, z2.s, #31
-; CHECK-COMPAT-NEXT: lsl z3.s, z3.s, #31
; CHECK-COMPAT-NEXT: asr z2.s, z2.s, #31
-; CHECK-COMPAT-NEXT: asr z3.s, z3.s, #31
-; CHECK-COMPAT-NEXT: cmpne p2.s, p0/z, z2.s, #0
-; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z3.s, #0
-; CHECK-COMPAT-NEXT: zip1 z3.d, z5.d, z4.d
-; CHECK-COMPAT-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
-; CHECK-COMPAT-NEXT: st1w { z0.s }, p2, [x0]
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; CHECK-COMPAT-NEXT: st2w { z0.s, z1.s }, p1, [x0]
; CHECK-COMPAT-NEXT: ret
;
; CHECK-STREAMING-LABEL: masked_store_factor2_128bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: adrp x8, .LCPI7_0
-; CHECK-STREAMING-NEXT: mov z4.s, z1.s[3]
-; CHECK-STREAMING-NEXT: mov z5.s, z0.s[3]
-; CHECK-STREAMING-NEXT: ldr q3, [x8, :lo12:.LCPI7_0]
-; CHECK-STREAMING-NEXT: mov z6.s, z1.s[2]
-; CHECK-STREAMING-NEXT: mov z7.s, z0.s[2]
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
-; CHECK-STREAMING-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-STREAMING-NEXT: mov x8, #4 // =0x4
-; CHECK-STREAMING-NEXT: tbl z3.h, { z2.h }, z3.h
-; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z2.h
-; CHECK-STREAMING-NEXT: zip1 z4.s, z5.s, z4.s
-; CHECK-STREAMING-NEXT: zip1 z5.s, z7.s, z6.s
; CHECK-STREAMING-NEXT: uunpklo z2.s, z2.h
-; CHECK-STREAMING-NEXT: uunpklo z3.s, z3.h
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
; CHECK-STREAMING-NEXT: lsl z2.s, z2.s, #31
-; CHECK-STREAMING-NEXT: lsl z3.s, z3.s, #31
; CHECK-STREAMING-NEXT: asr z2.s, z2.s, #31
-; CHECK-STREAMING-NEXT: asr z3.s, z3.s, #31
-; CHECK-STREAMING-NEXT: cmpne p2.s, p0/z, z2.s, #0
-; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z3.s, #0
-; CHECK-STREAMING-NEXT: zip1 z3.d, z5.d, z4.d
-; CHECK-STREAMING-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
-; CHECK-STREAMING-NEXT: st1w { z0.s }, p2, [x0]
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; CHECK-STREAMING-NEXT: st2w { z0.s, z1.s }, p1, [x0]
; CHECK-STREAMING-NEXT: ret
%interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
<4 x i32> %v0, <4 x i32> %v1)
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
index 133bac768d59e..70f02aeb91c6f 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
@@ -266,11 +266,9 @@ define void @alloc_v4i8_intrinsic(ptr %st_ptr) nounwind {
; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: ld1b { z0.h }, p0/z, [x20]
; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: mov z1.h, z0.h[2]
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: fmov w9, s1
-; CHECK-NEXT: mov z0.s, w8
-; CHECK-NEXT: mov z1.s, w9
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: ext z1.b, z1.b, z0.b, #8
; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
; CHECK-NEXT: st1b { z0.s }, p0, [x19]
; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
index 986b776a9a586..3d7e5eec7be63 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
@@ -194,7 +194,9 @@ define void @hang_when_merging_stores_after_legalisation_intrinsic(ptr %a, <2 x
; CHECK-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, s0
-; CHECK-NEXT: stp q0, q0, [x0]
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
@@ -219,15 +221,8 @@ define void @hang_when_merging_stores_after_legalisation_intrinsic(ptr %a, <2 x
define void @interleave_store_without_splat_intrinsic(ptr %a, <4 x i32> %v1, <4 x i32> %v2) {
; CHECK-LABEL: interleave_store_without_splat_intrinsic:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.s, z1.s[3]
-; CHECK-NEXT: mov z3.s, z0.s[3]
-; CHECK-NEXT: mov z4.s, z1.s[2]
-; CHECK-NEXT: mov z5.s, z0.s[2]
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: zip1 z2.s, z3.s, z2.s
-; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
-; CHECK-NEXT: zip1 z1.d, z3.d, z2.d
-; CHECK-NEXT: stp q0, q1, [x0]
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: interleave_store_without_splat_intrinsic:
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 4870243e0e4a4..3ad43f71dc2a1 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -1064,159 +1064,82 @@ for.body: ; preds = %for.body.preheader5
define void @loop2_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
-; CHECK-IAENABLED-LABEL: loop2_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: subs w8, w2, #1
-; CHECK-IAENABLED-NEXT: b.lt .LBB4_6
-; CHECK-IAENABLED-NEXT: // %bb.1: // %for.body.preheader
-; CHECK-IAENABLED-NEXT: cmp w8, #2
-; CHECK-IAENABLED-NEXT: b.ls .LBB4_3
-; CHECK-IAENABLED-NEXT: // %bb.2: // %vector.memcheck
-; CHECK-IAENABLED-NEXT: ubfiz x9, x8, #1, #32
-; CHECK-IAENABLED-NEXT: add x9, x9, #2
-; CHECK-IAENABLED-NEXT: add x10, x0, x9
-; CHECK-IAENABLED-NEXT: add x9, x1, x9, lsl #2
-; CHECK-IAENABLED-NEXT: cmp x10, x1
-; CHECK-IAENABLED-NEXT: ccmp x9, x0, #0, hi
-; CHECK-IAENABLED-NEXT: b.ls .LBB4_7
-; CHECK-IAENABLED-NEXT: .LBB4_3:
-; CHECK-IAENABLED-NEXT: mov w10, wzr
-; CHECK-IAENABLED-NEXT: mov x8, x1
-; CHECK-IAENABLED-NEXT: mov x9, x0
-; CHECK-IAENABLED-NEXT: .LBB4_4: // %for.body.preheader1
-; CHECK-IAENABLED-NEXT: movi d0, #0000000000000000
-; CHECK-IAENABLED-NEXT: mov w11, #1132396544 // =0x437f0000
-; CHECK-IAENABLED-NEXT: sub w10, w2, w10
-; CHECK-IAENABLED-NEXT: fmov s1, w11
-; CHECK-IAENABLED-NEXT: .LBB4_5: // %for.body
-; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IAENABLED-NEXT: ldp s2, s3, [x8], #8
-; CHECK-IAENABLED-NEXT: fcmp s2, s1
-; CHECK-IAENABLED-NEXT: fcsel s4, s1, s2, gt
-; CHECK-IAENABLED-NEXT: fcmp s2, #0.0
-; CHECK-IAENABLED-NEXT: fcsel s2, s0, s4, mi
-; CHECK-IAENABLED-NEXT: fcmp s3, s1
-; CHECK-IAENABLED-NEXT: fcsel s4, s1, s3, gt
-; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
-; CHECK-IAENABLED-NEXT: fcvtzs s2, s2
-; CHECK-IAENABLED-NEXT: fcsel s3, s0, s4, mi
-; CHECK-IAENABLED-NEXT: subs w10, w10, #1
-; CHECK-IAENABLED-NEXT: str b2, [x9]
-; CHECK-IAENABLED-NEXT: fcvtzs s3, s3
-; CHECK-IAENABLED-NEXT: stur b3, [x9, #1]
-; CHECK-IAENABLED-NEXT: add x9, x9, #2
-; CHECK-IAENABLED-NEXT: b.ne .LBB4_5
-; CHECK-IAENABLED-NEXT: .LBB4_6: // %for.cond.cleanup
-; CHECK-IAENABLED-NEXT: ret
-; CHECK-IAENABLED-NEXT: .LBB4_7: // %vector.ph
-; CHECK-IAENABLED-NEXT: add x11, x8, #1
-; CHECK-IAENABLED-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-IAENABLED-NEXT: and x10, x11, #0x1fffffffc
-; CHECK-IAENABLED-NEXT: dup v0.4s, w8
-; CHECK-IAENABLED-NEXT: and x12, x11, #0x1fffffffc
-; CHECK-IAENABLED-NEXT: add x8, x1, x10, lsl #3
-; CHECK-IAENABLED-NEXT: add x9, x0, x10, lsl #1
-; CHECK-IAENABLED-NEXT: .LBB4_8: // %vector.body
-; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x1], #32
-; CHECK-IAENABLED-NEXT: subs x12, x12, #4
-; CHECK-IAENABLED-NEXT: fcmgt v3.4s, v1.4s, v0.4s
-; CHECK-IAENABLED-NEXT: fcmgt v4.4s, v2.4s, v0.4s
-; CHECK-IAENABLED-NEXT: fcmlt v5.4s, v1.4s, #0.0
-; CHECK-IAENABLED-NEXT: bsl v3.16b, v0.16b, v1.16b
-; CHECK-IAENABLED-NEXT: bsl v4.16b, v0.16b, v2.16b
-; CHECK-IAENABLED-NEXT: fcmlt v1.4s, v2.4s, #0.0
-; CHECK-IAENABLED-NEXT: bic v2.16b, v3.16b, v5.16b
-; CHECK-IAENABLED-NEXT: bic v1.16b, v4.16b, v1.16b
-; CHECK-IAENABLED-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: fcvtzs v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: xtn v2.4h, v2.4s
-; CHECK-IAENABLED-NEXT: xtn v1.4h, v1.4s
-; CHECK-IAENABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
-; CHECK-IAENABLED-NEXT: str d1, [x0], #8
-; CHECK-IAENABLED-NEXT: b.ne .LBB4_8
-; CHECK-IAENABLED-NEXT: // %bb.9: // %middle.block
-; CHECK-IAENABLED-NEXT: cmp x11, x10
-; CHECK-IAENABLED-NEXT: b.ne .LBB4_4
-; CHECK-IAENABLED-NEXT: b .LBB4_6
-;
-; CHECK-IADISABLED-LABEL: loop2_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: subs w8, w2, #1
-; CHECK-IADISABLED-NEXT: b.lt .LBB4_6
-; CHECK-IADISABLED-NEXT: // %bb.1: // %for.body.preheader
-; CHECK-IADISABLED-NEXT: cmp w8, #2
-; CHECK-IADISABLED-NEXT: b.ls .LBB4_3
-; CHECK-IADISABLED-NEXT: // %bb.2: // %vector.memcheck
-; CHECK-IADISABLED-NEXT: ubfiz x9, x8, #1, #32
-; CHECK-IADISABLED-NEXT: add x9, x9, #2
-; CHECK-IADISABLED-NEXT: add x10, x0, x9
-; CHECK-IADISABLED-NEXT: add x9, x1, x9, lsl #2
-; CHECK-IADISABLED-NEXT: cmp x10, x1
-; CHECK-IADISABLED-NEXT: ccmp x9, x0, #0, hi
-; CHECK-IADISABLED-NEXT: b.ls .LBB4_7
-; CHECK-IADISABLED-NEXT: .LBB4_3:
-; CHECK-IADISABLED-NEXT: mov w10, wzr
-; CHECK-IADISABLED-NEXT: mov x8, x1
-; CHECK-IADISABLED-NEXT: mov x9, x0
-; CHECK-IADISABLED-NEXT: .LBB4_4: // %for.body.preheader1
-; CHECK-IADISABLED-NEXT: movi d0, #0000000000000000
-; CHECK-IADISABLED-NEXT: mov w11, #1132396544 // =0x437f0000
-; CHECK-IADISABLED-NEXT: sub w10, w2, w10
-; CHECK-IADISABLED-NEXT: fmov s1, w11
-; CHECK-IADISABLED-NEXT: .LBB4_5: // %for.body
-; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp s2, s3, [x8], #8
-; CHECK-IADISABLED-NEXT: fcmp s2, s1
-; CHECK-IADISABLED-NEXT: fcsel s4, s1, s2, gt
-; CHECK-IADISABLED-NEXT: fcmp s2, #0.0
-; CHECK-IADISABLED-NEXT: fcsel s2, s0, s4, mi
-; CHECK-IADISABLED-NEXT: fcmp s3, s1
-; CHECK-IADISABLED-NEXT: fcsel s4, s1, s3, gt
-; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
-; CHECK-IADISABLED-NEXT: fcvtzs s2, s2
-; CHECK-IADISABLED-NEXT: fcsel s3, s0, s4, mi
-; CHECK-IADISABLED-NEXT: subs w10, w10, #1
-; CHECK-IADISABLED-NEXT: str b2, [x9]
-; CHECK-IADISABLED-NEXT: fcvtzs s3, s3
-; CHECK-IADISABLED-NEXT: stur b3, [x9, #1]
-; CHECK-IADISABLED-NEXT: add x9, x9, #2
-; CHECK-IADISABLED-NEXT: b.ne .LBB4_5
-; CHECK-IADISABLED-NEXT: .LBB4_6: // %for.cond.cleanup
-; CHECK-IADISABLED-NEXT: ret
-; CHECK-IADISABLED-NEXT: .LBB4_7: // %vector.ph
-; CHECK-IADISABLED-NEXT: add x11, x8, #1
-; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
-; CHECK-IADISABLED-NEXT: dup v0.4s, w8
-; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
-; CHECK-IADISABLED-NEXT: add x8, x1, x10, lsl #3
-; CHECK-IADISABLED-NEXT: add x9, x0, x10, lsl #1
-; CHECK-IADISABLED-NEXT: .LBB4_8: // %vector.body
-; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q2, q1, [x1], #32
-; CHECK-IADISABLED-NEXT: subs x12, x12, #4
-; CHECK-IADISABLED-NEXT: uzp1 v3.4s, v2.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v3.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v3.16b
-; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
-; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v1.16b
-; CHECK-IADISABLED-NEXT: fcmlt v1.4s, v1.4s, #0.0
-; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v3.16b
-; CHECK-IADISABLED-NEXT: bic v1.16b, v4.16b, v1.16b
-; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fcvtzs v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: xtn v2.4h, v2.4s
-; CHECK-IADISABLED-NEXT: xtn v1.4h, v1.4s
-; CHECK-IADISABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
-; CHECK-IADISABLED-NEXT: str d1, [x0], #8
-; CHECK-IADISABLED-NEXT: b.ne .LBB4_8
-; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
-; CHECK-IADISABLED-NEXT: cmp x11, x10
-; CHECK-IADISABLED-NEXT: b.ne .LBB4_4
-; CHECK-IADISABLED-NEXT: b .LBB4_6
+; CHECK-LABEL: loop2_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: subs w8, w2, #1
+; CHECK-NEXT: b.lt .LBB4_6
+; CHECK-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-NEXT: cmp w8, #2
+; CHECK-NEXT: b.ls .LBB4_3
+; CHECK-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-NEXT: ubfiz x9, x8, #1, #32
+; CHECK-NEXT: add x9, x9, #2
+; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: add x9, x1, x9, lsl #2
+; CHECK-NEXT: cmp x10, x1
+; CHECK-NEXT: ccmp x9, x0, #0, hi
+; CHECK-NEXT: b.ls .LBB4_7
+; CHECK-NEXT: .LBB4_3:
+; CHECK-NEXT: mov w10, wzr
+; CHECK-NEXT: mov x8, x1
+; CHECK-NEXT: mov x9, x0
+; CHECK-NEXT: .LBB4_4: // %for.body.preheader1
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT: sub w10, w2, w10
+; CHECK-NEXT: fmov s1, w11
+; CHECK-NEXT: .LBB4_5: // %for.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldp s2, s3, [x8], #8
+; CHECK-NEXT: fcmp s2, s1
+; CHECK-NEXT: fcsel s4, s1, s2, gt
+; CHECK-NEXT: fcmp s2, #0.0
+; CHECK-NEXT: fcsel s2, s0, s4, mi
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: fcsel s4, s1, s3, gt
+; CHECK-NEXT: fcmp s3, #0.0
+; CHECK-NEXT: fcvtzs s2, s2
+; CHECK-NEXT: fcsel s3, s0, s4, mi
+; CHECK-NEXT: subs w10, w10, #1
+; CHECK-NEXT: str b2, [x9]
+; CHECK-NEXT: fcvtzs s3, s3
+; CHECK-NEXT: stur b3, [x9, #1]
+; CHECK-NEXT: add x9, x9, #2
+; CHECK-NEXT: b.ne .LBB4_5
+; CHECK-NEXT: .LBB4_6: // %for.cond.cleanup
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB4_7: // %vector.ph
+; CHECK-NEXT: add x11, x8, #1
+; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-NEXT: dup v0.4s, w8
+; CHECK-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-NEXT: add x8, x1, x10, lsl #3
+; CHECK-NEXT: add x9, x0, x10, lsl #1
+; CHECK-NEXT: .LBB4_8: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x1], #32
+; CHECK-NEXT: subs x12, x12, #4
+; CHECK-NEXT: fcmgt v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: fcmgt v4.4s, v2.4s, v0.4s
+; CHECK-NEXT: fcmlt v5.4s, v1.4s, #0.0
+; CHECK-NEXT: bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT: bsl v4.16b, v0.16b, v2.16b
+; CHECK-NEXT: fcmlt v1.4s, v2.4s, #0.0
+; CHECK-NEXT: bic v2.16b, v3.16b, v5.16b
+; CHECK-NEXT: bic v1.16b, v4.16b, v1.16b
+; CHECK-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-NEXT: xtn v2.4h, v2.4s
+; CHECK-NEXT: xtn v1.4h, v1.4s
+; CHECK-NEXT: zip1 v1.8h, v2.8h, v1.8h
+; CHECK-NEXT: xtn v1.8b, v1.8h
+; CHECK-NEXT: str d1, [x0], #8
+; CHECK-NEXT: b.ne .LBB4_8
+; CHECK-NEXT: // %bb.9: // %middle.block
+; CHECK-NEXT: cmp x11, x10
+; CHECK-NEXT: b.ne .LBB4_4
+; CHECK-NEXT: b .LBB4_6
entry:
%cmp19 = icmp sgt i32 %width, 0
br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
diff --git a/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
index b63448c643074..9d884c150f956 100644
--- a/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
@@ -1,33 +1,18 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s
define void @aarch64_vector_deinterleave_idx_ld2(ptr %ptr, i64 %idx) {
-; CHECK-IAENABLED-LABEL: aarch64_vector_deinterleave_idx_ld2:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: lsl x8, x1, #2
-; CHECK-IAENABLED-NEXT: and x9, x8, #0xfffffffffffffff0
-; CHECK-IAENABLED-NEXT: add x8, x8, #16
-; CHECK-IAENABLED-NEXT: add x10, x0, x9
-; CHECK-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
-; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x10]
-; CHECK-IAENABLED-NEXT: str q0, [x0, x9]
-; CHECK-IAENABLED-NEXT: str q1, [x0, x8]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: aarch64_vector_deinterleave_idx_ld2:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: lsl x8, x1, #2
-; CHECK-IADISABLED-NEXT: and x9, x8, #0xfffffffffffffff0
-; CHECK-IADISABLED-NEXT: add x8, x8, #16
-; CHECK-IADISABLED-NEXT: add x9, x0, x9
-; CHECK-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x9]
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: str q2, [x9]
-; CHECK-IADISABLED-NEXT: str q0, [x0, x8]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: aarch64_vector_deinterleave_idx_ld2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsl x8, x1, #2
+; CHECK-NEXT: and x9, x8, #0xfffffffffffffff0
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; CHECK-NEXT: str q0, [x0, x9]
+; CHECK-NEXT: str q1, [x0, x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index d3985991af531..10eee0fcf3ba7 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -1,24 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s
+
define void @aarch64_vector_interleave_idx_st2(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1) {
-; CHECK-IAENABLED-LABEL: aarch64_vector_interleave_idx_st2:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: lsr x8, x1, #2
-; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-IAENABLED-NEXT: add x8, x0, x8, lsl #4
-; CHECK-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x8]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: aarch64_vector_interleave_idx_st2:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: lsr x8, x1, #2
-; CHECK-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: add x8, x0, x8, lsl #4
-; CHECK-IADISABLED-NEXT: stp q0, q2, [x8]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: aarch64_vector_interleave_idx_st2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsr x8, x1, #2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT: add x8, x0, x8, lsl #4
+; CHECK-NEXT: st2 { v0.4s, v1.4s }, [x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
diff --git a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
index 963c28aeba32b..fac725dd74999 100644
--- a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
@@ -1075,12 +1075,10 @@ define void @vld2_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture
; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: .LBB17_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldp q1, q0, [x0], #32
-; CHECK-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; CHECK-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: fmul v1.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v1.4s, v0.4s, v0.4s
-; CHECK-NEXT: str q1, [x1, x8]
+; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; CHECK-NEXT: fmul v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v2.4s, v1.4s, v1.4s
+; CHECK-NEXT: str q2, [x1, x8]
; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
; CHECK-NEXT: b.ne .LBB17_1
@@ -1161,47 +1159,25 @@ while.end: ; preds = %vector.body
define void @vld4_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-IAENABLED-LABEL: vld4_intrinsic:
-; CHECK-IAENABLED: .Lfunc_begin19:
-; CHECK-IAENABLED-NEXT: .cfi_startproc
-; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: mov x8, xzr
-; CHECK-IAENABLED-NEXT: .LBB19_1: // %vector.body
-; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IAENABLED-NEXT: add x9, x1, x8
-; CHECK-IAENABLED-NEXT: add x8, x8, #32
-; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
-; CHECK-IAENABLED-NEXT: b.ne .LBB19_1
-; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld4_intrinsic:
-; CHECK-IADISABLED: .Lfunc_begin19:
-; CHECK-IADISABLED-NEXT: .cfi_startproc
-; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov x8, xzr
-; CHECK-IADISABLED-NEXT: .LBB19_1: // %vector.body
-; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IADISABLED-NEXT: add x9, x1, x8
-; CHECK-IADISABLED-NEXT: add x8, x8, #32
-; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-IADISABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-IADISABLED-NEXT: zip2 v0.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: stp q1, q0, [x9]
-; CHECK-IADISABLED-NEXT: b.ne .LBB19_1
-; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: vld4_intrinsic:
+; CHECK: .Lfunc_begin19:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB19_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-NEXT: add x9, x1, x8
+; CHECK-NEXT: add x8, x8, #32
+; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-NEXT: b.ne .LBB19_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
entry:
br label %vector.body
@@ -1248,16 +1224,12 @@ define void @twosrc_intrinsic(ptr nocapture readonly %pSrc, ptr nocapture readon
; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: add x10, x1, x8
; CHECK-NEXT: add x8, x8, #32
-; CHECK-NEXT: ldp q1, q0, [x9]
+; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x9]
; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-NEXT: ldp q3, q2, [x10]
-; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
-; CHECK-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: uzp1 v5.4s, v3.4s, v2.4s
-; CHECK-NEXT: uzp2 v1.4s, v3.4s, v2.4s
-; CHECK-NEXT: fmul v2.4s, v5.4s, v4.4s
-; CHECK-NEXT: fmla v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: str q2, [x2], #16
+; CHECK-NEXT: ld2 { v2.4s, v3.4s }, [x10]
+; CHECK-NEXT: fmul v4.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmla v4.4s, v1.4s, v3.4s
+; CHECK-NEXT: str q4, [x2], #16
; CHECK-NEXT: b.ne .LBB20_1
; CHECK-NEXT: // %bb.2: // %while.end
; CHECK-NEXT: ret
@@ -1295,41 +1267,22 @@ while.end: ; preds = %vector.body
define void @vld2_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-IAENABLED-LABEL: vld2_multiuse_intrinsic:
-; CHECK-IAENABLED: .Lfunc_begin21:
-; CHECK-IAENABLED-NEXT: .cfi_startproc
-; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: mov x8, xzr
-; CHECK-IAENABLED-NEXT: .LBB21_1: // %vector.body
-; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
-; CHECK-IAENABLED-NEXT: fmul v2.4s, v0.4s, v0.4s
-; CHECK-IAENABLED-NEXT: fmla v2.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: str q2, [x1, x8]
-; CHECK-IAENABLED-NEXT: add x8, x8, #16
-; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-IAENABLED-NEXT: b.ne .LBB21_1
-; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld2_multiuse_intrinsic:
-; CHECK-IADISABLED: .Lfunc_begin21:
-; CHECK-IADISABLED-NEXT: .cfi_startproc
-; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov x8, xzr
-; CHECK-IADISABLED-NEXT: .LBB21_1: // %vector.body
-; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0], #32
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmla v1.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: str q1, [x1, x8]
-; CHECK-IADISABLED-NEXT: add x8, x8, #16
-; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-IADISABLED-NEXT: b.ne .LBB21_1
-; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: vld2_multiuse_intrinsic:
+; CHECK: .Lfunc_begin21:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB21_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; CHECK-NEXT: fmul v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v2.4s, v1.4s, v1.4s
+; CHECK-NEXT: str q2, [x1, x8]
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-NEXT: b.ne .LBB21_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
entry:
br label %vector.body
@@ -1402,47 +1355,25 @@ while.end: ; preds = %vector.body
define void @vld4_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-IAENABLED-LABEL: vld4_multiuse_intrinsic:
-; CHECK-IAENABLED: .Lfunc_begin23:
-; CHECK-IAENABLED-NEXT: .cfi_startproc
-; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: mov x8, xzr
-; CHECK-IAENABLED-NEXT: .LBB23_1: // %vector.body
-; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IAENABLED-NEXT: add x9, x1, x8
-; CHECK-IAENABLED-NEXT: add x8, x8, #32
-; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
-; CHECK-IAENABLED-NEXT: b.ne .LBB23_1
-; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld4_multiuse_intrinsic:
-; CHECK-IADISABLED: .Lfunc_begin23:
-; CHECK-IADISABLED-NEXT: .cfi_startproc
-; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov x8, xzr
-; CHECK-IADISABLED-NEXT: .LBB23_1: // %vector.body
-; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-IADISABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IADISABLED-NEXT: add x9, x1, x8
-; CHECK-IADISABLED-NEXT: add x8, x8, #32
-; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-IADISABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-IADISABLED-NEXT: zip2 v0.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: stp q1, q0, [x9]
-; CHECK-IADISABLED-NEXT: b.ne .LBB23_1
-; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: vld4_multiuse_intrinsic:
+; CHECK: .Lfunc_begin23:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB23_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-NEXT: add x9, x1, x8
+; CHECK-NEXT: add x8, x8, #32
+; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-NEXT: b.ne .LBB23_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
entry:
br label %vector.body
@@ -1534,18 +1465,18 @@ define void @transpose_s16_8x8_simpler2_intrinsic(ptr nocapture noundef %a) {
; CHECK-NEXT: .cfi_startproc
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ldp q2, q3, [x0, #64]
-; CHECK-NEXT: ldp q4, q5, [x0, #96]
+; CHECK-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-NEXT: ldp q6, q7, [x0, #96]
; CHECK-NEXT: trn1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: zip1 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT: trn1 v2.8h, v4.8h, v5.8h
-; CHECK-NEXT: ldp q3, q4, [x0, #32]
-; CHECK-NEXT: zip1 v3.8h, v3.8h, v4.8h
-; CHECK-NEXT: dup v1.4s, v1.s[0]
-; CHECK-NEXT: uzp1 v2.4s, v0.4s, v2.4s
-; CHECK-NEXT: zip1 v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: zip2 v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: str q1, [x0]
+; CHECK-NEXT: zip1 v2.8h, v4.8h, v5.8h
+; CHECK-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: str q2, [x0]
; CHECK-NEXT: str q0, [x0, #64]
; CHECK-NEXT: ret
entry:
@@ -1684,26 +1615,14 @@ define void @transpose_s16_8x8_intrinsic(ptr nocapture noundef %0, ptr nocapture
}
define void @store_factor2_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-IAENABLED-LABEL: store_factor2_intrinsic:
-; CHECK-IAENABLED: .Lfunc_begin27:
-; CHECK-IAENABLED-NEXT: .cfi_startproc
-; CHECK-IAENABLED-NEXT: // %bb.0:
-; CHECK-IAENABLED-NEXT: trn1 v2.4s, v0.4s, v1.4s
-; CHECK-IAENABLED-NEXT: trn1 v3.4s, v1.4s, v0.4s
-; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: store_factor2_intrinsic:
-; CHECK-IADISABLED: .Lfunc_begin27:
-; CHECK-IADISABLED-NEXT: .cfi_startproc
-; CHECK-IADISABLED-NEXT: // %bb.0:
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: ext v0.16b, v2.16b, v0.16b, #8
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v2.4s, v0.4s
-; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: store_factor2_intrinsic:
+; CHECK: .Lfunc_begin27:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
diff --git a/llvm/test/CodeGen/X86/vector-interleave-intrinsics.ll b/llvm/test/CodeGen/X86/vector-interleave-intrinsics.ll
new file mode 100644
index 0000000000000..5046e822918cc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-interleave-intrinsics.ll
@@ -0,0 +1,576 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @interleave2(ptr %dst, <4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: interleave2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm0, %xmm2
+; CHECK-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm0, 16(%rdi)
+; CHECK-NEXT: movaps %xmm2, (%rdi)
+; CHECK-NEXT: retq
+ %result = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave2(ptr %src, ptr %dst0, ptr %dst1) {
+; CHECK-LABEL: deinterleave2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps (%rdi), %xmm0
+; CHECK-NEXT: movaps 16(%rdi), %xmm1
+; CHECK-NEXT: movaps %xmm0, %xmm2
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[1,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; CHECK-NEXT: movaps %xmm0, (%rsi)
+; CHECK-NEXT: movaps %xmm2, (%rdx)
+; CHECK-NEXT: retq
+ %wide = load <8 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32> } %result, 1
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ ret void
+}
+
+define void @interleave3(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: interleave3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm3
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[1,3]
+; CHECK-NEXT: movaps %xmm0, %xmm4
+; CHECK-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[0,2]
+; CHECK-NEXT: movaps %xmm0, %xmm3
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3],xmm1[3,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm2[2,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm3[0,2]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]
+; CHECK-NEXT: movaps %xmm2, 16(%rdi)
+; CHECK-NEXT: movaps %xmm4, (%rdi)
+; CHECK-NEXT: movaps %xmm0, 32(%rdi)
+; CHECK-NEXT: retq
+ %result = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave3(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2) {
+; CHECK-LABEL: deinterleave3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movdqa (%rdi), %xmm0
+; CHECK-NEXT: movdqa 16(%rdi), %xmm1
+; CHECK-NEXT: movaps 32(%rdi), %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[0,3]
+; CHECK-NEXT: movdqa %xmm1, %xmm3
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm2[2,3]
+; CHECK-NEXT: movdqa %xmm0, %xmm5
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm1[0,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2],xmm3[0,2]
+; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[1,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,2]
+; CHECK-NEXT: movaps %xmm0, (%rsi)
+; CHECK-NEXT: movaps %xmm5, (%rdx)
+; CHECK-NEXT: movaps %xmm4, (%rcx)
+; CHECK-NEXT: retq
+ %wide = load <12 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ ret void
+}
+
+define void @interleave4(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; CHECK-LABEL: interleave4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm4
+; CHECK-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; CHECK-NEXT: movaps %xmm0, %xmm5
+; CHECK-NEXT: unpcklps {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; CHECK-NEXT: movaps %xmm5, %xmm6
+; CHECK-NEXT: movlhps {{.*#+}} xmm6 = xmm6[0],xmm4[0]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm4[1]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm0, %xmm1
+; CHECK-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
+; CHECK-NEXT: movaps %xmm0, 48(%rdi)
+; CHECK-NEXT: movaps %xmm1, 32(%rdi)
+; CHECK-NEXT: movaps %xmm5, 16(%rdi)
+; CHECK-NEXT: movaps %xmm6, (%rdi)
+; CHECK-NEXT: retq
+ %result = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
+ store <16 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave4(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2, ptr %dst3) {
+; CHECK-LABEL: deinterleave4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps (%rdi), %xmm0
+; CHECK-NEXT: movaps 16(%rdi), %xmm1
+; CHECK-NEXT: movaps 32(%rdi), %xmm2
+; CHECK-NEXT: movaps 48(%rdi), %xmm3
+; CHECK-NEXT: movaps %xmm2, %xmm4
+; CHECK-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: movaps %xmm0, %xmm5
+; CHECK-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm5, %xmm6
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm4[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm5 = xmm5[0],xmm4[0]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT: movaps %xmm0, %xmm1
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: movaps %xmm0, (%rsi)
+; CHECK-NEXT: movaps %xmm1, (%rdx)
+; CHECK-NEXT: movaps %xmm5, (%rcx)
+; CHECK-NEXT: movaps %xmm6, (%r8)
+; CHECK-NEXT: retq
+ %wide = load <16 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 3
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ store <4 x i32> %v3, ptr %dst3
+ ret void
+}
+
+define void @interleave5(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4) {
+; CHECK-LABEL: interleave5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm5
+; CHECK-NEXT: unpcklps {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; CHECK-NEXT: movaps %xmm0, %xmm6
+; CHECK-NEXT: unpcklps {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm6 = xmm6[0],xmm5[0]
+; CHECK-NEXT: movaps %xmm0, %xmm5
+; CHECK-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm4, %xmm7
+; CHECK-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,1],xmm3[1,1]
+; CHECK-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm5[0,1]
+; CHECK-NEXT: movaps %xmm4, %xmm5
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,1],xmm0[3,3]
+; CHECK-NEXT: movaps %xmm4, %xmm8
+; CHECK-NEXT: shufps {{.*#+}} xmm8 = xmm8[3,3],xmm3[3,3]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm5[0,2]
+; CHECK-NEXT: movaps %xmm1, %xmm5
+; CHECK-NEXT: unpckhps {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,3],xmm8[2,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; CHECK-NEXT: movss {{.*#+}} xmm0 = xmm4[0],xmm0[1,2,3]
+; CHECK-NEXT: movaps %xmm5, 64(%rdi)
+; CHECK-NEXT: movaps %xmm3, 48(%rdi)
+; CHECK-NEXT: movaps %xmm7, 32(%rdi)
+; CHECK-NEXT: movaps %xmm6, (%rdi)
+; CHECK-NEXT: movaps %xmm0, 16(%rdi)
+; CHECK-NEXT: retq
+ %result = call <20 x i32> @llvm.vector.interleave5.v20i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4)
+ store <20 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave5(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2, ptr %dst3, ptr %dst4) {
+; CHECK-LABEL: deinterleave5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps 64(%rdi), %xmm1
+; CHECK-NEXT: movdqa (%rdi), %xmm0
+; CHECK-NEXT: movdqa 16(%rdi), %xmm4
+; CHECK-NEXT: movdqa 32(%rdi), %xmm2
+; CHECK-NEXT: movdqa 48(%rdi), %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm4[2,2,3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm8 = xmm4[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; CHECK-NEXT: movdqa %xmm3, %xmm5
+; CHECK-NEXT: movdqa %xmm3, %xmm9
+; CHECK-NEXT: pshufd {{.*#+}} xmm10 = xmm2[2,2,2,2]
+; CHECK-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm3[2],xmm10[3],xmm3[3]
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[3,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[0,2]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm1[2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm5[0,2]
+; CHECK-NEXT: movdqa %xmm0, %xmm5
+; CHECK-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm1[1,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm9[0,2]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1],xmm1[0,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; CHECK-NEXT: movsd {{.*#+}} xmm10 = xmm0[0],xmm10[1]
+; CHECK-NEXT: movapd %xmm10, (%rsi)
+; CHECK-NEXT: movaps %xmm1, (%rdx)
+; CHECK-NEXT: movaps %xmm5, (%rcx)
+; CHECK-NEXT: movaps %xmm3, (%r8)
+; CHECK-NEXT: movaps %xmm4, (%r9)
+; CHECK-NEXT: retq
+ %wide = load <20 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave5.v20i32(<20 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 3
+ %v4 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 4
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ store <4 x i32> %v3, ptr %dst3
+ store <4 x i32> %v4, ptr %dst4
+ ret void
+}
+
+define void @interleave6(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5) {
+; CHECK-LABEL: interleave6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm6
+; CHECK-NEXT: unpcklps {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
+; CHECK-NEXT: movaps %xmm0, %xmm7
+; CHECK-NEXT: unpcklps {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; CHECK-NEXT: movaps %xmm5, %xmm8
+; CHECK-NEXT: movlhps {{.*#+}} xmm8 = xmm8[0],xmm4[0]
+; CHECK-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm7[2,3]
+; CHECK-NEXT: movlhps {{.*#+}} xmm7 = xmm7[0],xmm6[0]
+; CHECK-NEXT: movaps %xmm5, %xmm9
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,1],xmm4[1,1]
+; CHECK-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,3],xmm9[2,0]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm5, %xmm1
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; CHECK-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,3],xmm4[3,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm5[2,0]
+; CHECK-NEXT: movaps %xmm2, 80(%rdi)
+; CHECK-NEXT: movaps %xmm1, 64(%rdi)
+; CHECK-NEXT: movaps %xmm0, 48(%rdi)
+; CHECK-NEXT: movaps %xmm6, 32(%rdi)
+; CHECK-NEXT: movaps %xmm8, 16(%rdi)
+; CHECK-NEXT: movaps %xmm7, (%rdi)
+; CHECK-NEXT: retq
+ %result = call <24 x i32> @llvm.vector.interleave6.v24i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5)
+ store <24 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave6(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2, ptr %dst3, ptr %dst4, ptr %dst5) {
+; CHECK-LABEL: deinterleave6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movdqa (%rdi), %xmm0
+; CHECK-NEXT: movdqa 16(%rdi), %xmm6
+; CHECK-NEXT: movdqa 32(%rdi), %xmm4
+; CHECK-NEXT: movapd 48(%rdi), %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm4[3,3,3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; CHECK-NEXT: movdqa 64(%rdi), %xmm3
+; CHECK-NEXT: movdqa 80(%rdi), %xmm5
+; CHECK-NEXT: pshufd {{.*#+}} xmm8 = xmm5[2,3,2,3]
+; CHECK-NEXT: movdqa %xmm3, %xmm1
+; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
+; CHECK-NEXT: movsd {{.*#+}} xmm1 = xmm7[0],xmm1[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm8 = xmm6[3,3,3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm9 = xmm6[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,2,3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm10 = xmm3[0,0,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
+; CHECK-NEXT: movsd {{.*#+}} xmm10 = xmm6[0],xmm10[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,1,1,1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; CHECK-NEXT: movapd %xmm2, %xmm6
+; CHECK-NEXT: movsd {{.*#+}} xmm6 = xmm5[0],xmm6[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm2[3,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm5[2,0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,0]
+; CHECK-NEXT: movdqa %xmm3, %xmm4
+; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,1],xmm2[1,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,3],xmm2[0,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
+; CHECK-NEXT: movaps %xmm0, (%rsi)
+; CHECK-NEXT: movaps %xmm6, (%rdx)
+; CHECK-NEXT: movaps %xmm5, (%rcx)
+; CHECK-NEXT: movaps %xmm7, (%r8)
+; CHECK-NEXT: movapd %xmm10, (%r9)
+; CHECK-NEXT: movapd %xmm1, (%rax)
+; CHECK-NEXT: retq
+ %wide = load <24 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave6.v24i32(<24 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 3
+ %v4 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 4
+ %v5 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 5
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ store <4 x i32> %v3, ptr %dst3
+ store <4 x i32> %v4, ptr %dst4
+ store <4 x i32> %v5, ptr %dst5
+ ret void
+}
+
+define void @interleave7(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5, <4 x i32> %v6) {
+; CHECK-LABEL: interleave7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm8
+; CHECK-NEXT: unpcklps {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1]
+; CHECK-NEXT: movaps %xmm0, %xmm7
+; CHECK-NEXT: unpcklps {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm7 = xmm7[0],xmm8[0]
+; CHECK-NEXT: movaps %xmm6, %xmm9
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm0[1,3]
+; CHECK-NEXT: movaps %xmm4, %xmm10
+; CHECK-NEXT: shufps {{.*#+}} xmm10 = xmm10[1,1],xmm3[1,1]
+; CHECK-NEXT: movaps %xmm4, %xmm11
+; CHECK-NEXT: movaps %xmm2, %xmm8
+; CHECK-NEXT: unpckhps {{.*#+}} xmm8 = xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm9[0,2]
+; CHECK-NEXT: movaps %xmm1, %xmm9
+; CHECK-NEXT: unpcklps {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,3],xmm10[2,0]
+; CHECK-NEXT: movaps %xmm0, %xmm10
+; CHECK-NEXT: unpckhps {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm5, %xmm12
+; CHECK-NEXT: unpcklps {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,3],xmm10[0,1]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm11 = xmm11[2],xmm5[2],xmm11[3],xmm5[3]
+; CHECK-NEXT: movlhps {{.*#+}} xmm8 = xmm8[0],xmm11[0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm10 = xmm6[2,3,2,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm6 = xmm6[3,3],xmm5[3,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,3],xmm6[2,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3],xmm1[3,3]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[2,0]
+; CHECK-NEXT: movss {{.*#+}} xmm0 = xmm10[0],xmm0[1,2,3]
+; CHECK-NEXT: movaps %xmm3, 96(%rdi)
+; CHECK-NEXT: movaps %xmm8, 64(%rdi)
+; CHECK-NEXT: movaps %xmm12, 48(%rdi)
+; CHECK-NEXT: movaps %xmm9, 32(%rdi)
+; CHECK-NEXT: movaps %xmm4, 16(%rdi)
+; CHECK-NEXT: movaps %xmm7, (%rdi)
+; CHECK-NEXT: movaps %xmm0, 80(%rdi)
+; CHECK-NEXT: retq
+ %result = call <28 x i32> @llvm.vector.interleave7.v28i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5, <4 x i32> %v6)
+ store <28 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave7(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2, ptr %dst3, ptr %dst4, ptr %dst5, ptr %dst6) {
+; CHECK-LABEL: deinterleave7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: movaps 80(%rdi), %xmm0
+; CHECK-NEXT: movaps 96(%rdi), %xmm7
+; CHECK-NEXT: movaps %xmm7, %xmm3
+; CHECK-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm0[0,3]
+; CHECK-NEXT: movdqa (%rdi), %xmm2
+; CHECK-NEXT: movdqa 16(%rdi), %xmm8
+; CHECK-NEXT: movdqa 32(%rdi), %xmm6
+; CHECK-NEXT: movdqa 48(%rdi), %xmm4
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm8[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,0]
+; CHECK-NEXT: movaps 64(%rdi), %xmm3
+; CHECK-NEXT: movaps %xmm7, %xmm9
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,1],xmm3[3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm8[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm9[2,0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,0,1,1]
+; CHECK-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm3[2,0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm10 = xmm6[3,3,3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm11 = xmm8[3,3,3,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,1],xmm7[2,0]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm2[2,2,3,3]
+; CHECK-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm10 = xmm3[0,0,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1]
+; CHECK-NEXT: movsd {{.*#+}} xmm3 = xmm7[0],xmm3[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[1,1,1,1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm9 = xmm2[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm0[2,3,2,3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
+; CHECK-NEXT: movsd {{.*#+}} xmm10 = xmm9[0],xmm10[1]
+; CHECK-NEXT: movaps %xmm0, %xmm7
+; CHECK-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,1],xmm4[3,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm9 = xmm2[1,1,1,1]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm9 = xmm9[0,1],xmm7[2,0]
+; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm4[2,0]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1]
+; CHECK-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,0]
+; CHECK-NEXT: movaps %xmm2, (%rsi)
+; CHECK-NEXT: movaps %xmm9, (%rdx)
+; CHECK-NEXT: movapd %xmm10, (%rcx)
+; CHECK-NEXT: movapd %xmm3, (%r8)
+; CHECK-NEXT: movaps %xmm8, (%r9)
+; CHECK-NEXT: movaps %xmm5, (%r10)
+; CHECK-NEXT: movaps %xmm1, (%rax)
+; CHECK-NEXT: retq
+ %wide = load <28 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave7.v28i32(<28 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 3
+ %v4 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 4
+ %v5 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 5
+ %v6 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 6
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ store <4 x i32> %v3, ptr %dst3
+ store <4 x i32> %v4, ptr %dst4
+ store <4 x i32> %v5, ptr %dst5
+ store <4 x i32> %v6, ptr %dst6
+ ret void
+}
+
+define void @interleave8(ptr %dst, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5, <4 x i32> %v6, <4 x i32> %v7) {
+; CHECK-LABEL: interleave8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movaps %xmm2, %xmm10
+; CHECK-NEXT: unpcklps {{.*#+}} xmm10 = xmm10[0],xmm3[0],xmm10[1],xmm3[1]
+; CHECK-NEXT: movaps %xmm0, %xmm9
+; CHECK-NEXT: unpcklps {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
+; CHECK-NEXT: movaps %xmm9, %xmm8
+; CHECK-NEXT: movlhps {{.*#+}} xmm8 = xmm8[0],xmm10[0]
+; CHECK-NEXT: movaps %xmm6, %xmm11
+; CHECK-NEXT: unpcklps {{.*#+}} xmm11 = xmm11[0],xmm7[0],xmm11[1],xmm7[1]
+; CHECK-NEXT: movaps %xmm4, %xmm12
+; CHECK-NEXT: unpcklps {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1]
+; CHECK-NEXT: movaps %xmm12, %xmm13
+; CHECK-NEXT: movlhps {{.*#+}} xmm13 = xmm13[0],xmm11[0]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm9 = xmm9[1],xmm10[1]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm12 = xmm12[1],xmm11[1]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; CHECK-NEXT: movaps %xmm0, %xmm1
+; CHECK-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; CHECK-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; CHECK-NEXT: movaps %xmm4, %xmm3
+; CHECK-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm6[1]
+; CHECK-NEXT: movaps %xmm4, 112(%rdi)
+; CHECK-NEXT: movaps %xmm0, 96(%rdi)
+; CHECK-NEXT: movaps %xmm3, 80(%rdi)
+; CHECK-NEXT: movaps %xmm1, 64(%rdi)
+; CHECK-NEXT: movaps %xmm12, 48(%rdi)
+; CHECK-NEXT: movaps %xmm9, 32(%rdi)
+; CHECK-NEXT: movaps %xmm13, 16(%rdi)
+; CHECK-NEXT: movaps %xmm8, (%rdi)
+; CHECK-NEXT: retq
+ %result = call <32 x i32> @llvm.vector.interleave8.v32i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, <4 x i32> %v4, <4 x i32> %v5, <4 x i32> %v6, <4 x i32> %v7)
+ store <32 x i32> %result, ptr %dst
+ ret void
+}
+
+define void @deinterleave8(ptr %src, ptr %dst0, ptr %dst1, ptr %dst2, ptr %dst3, ptr %dst4, ptr %dst5, ptr %dst6, ptr %dst7) {
+; CHECK-LABEL: deinterleave8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT: movaps 112(%rdi), %xmm4
+; CHECK-NEXT: movaps 80(%rdi), %xmm5
+; CHECK-NEXT: movaps %xmm5, %xmm6
+; CHECK-NEXT: unpckhps {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; CHECK-NEXT: movaps (%rdi), %xmm3
+; CHECK-NEXT: movaps 16(%rdi), %xmm0
+; CHECK-NEXT: movaps 32(%rdi), %xmm7
+; CHECK-NEXT: movaps 48(%rdi), %xmm8
+; CHECK-NEXT: movaps %xmm0, %xmm1
+; CHECK-NEXT: unpckhps {{.*#+}} xmm1 = xmm1[2],xmm8[2],xmm1[3],xmm8[3]
+; CHECK-NEXT: movaps %xmm1, %xmm2
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm6[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm6[0]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; CHECK-NEXT: movaps %xmm0, %xmm4
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm5[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; CHECK-NEXT: movaps 96(%rdi), %xmm5
+; CHECK-NEXT: movaps 64(%rdi), %xmm6
+; CHECK-NEXT: movaps %xmm6, %xmm8
+; CHECK-NEXT: unpckhps {{.*#+}} xmm8 = xmm8[2],xmm5[2],xmm8[3],xmm5[3]
+; CHECK-NEXT: movaps %xmm3, %xmm9
+; CHECK-NEXT: unpckhps {{.*#+}} xmm9 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; CHECK-NEXT: movaps %xmm9, %xmm10
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm10 = xmm10[1],xmm8[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm9 = xmm9[0],xmm8[0]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; CHECK-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]
+; CHECK-NEXT: movaps %xmm3, %xmm5
+; CHECK-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm6[1]
+; CHECK-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; CHECK-NEXT: movaps %xmm3, (%rsi)
+; CHECK-NEXT: movaps %xmm5, (%rdx)
+; CHECK-NEXT: movaps %xmm9, (%rcx)
+; CHECK-NEXT: movaps %xmm10, (%r8)
+; CHECK-NEXT: movaps %xmm0, (%r9)
+; CHECK-NEXT: movaps %xmm4, (%r11)
+; CHECK-NEXT: movaps %xmm1, (%r10)
+; CHECK-NEXT: movaps %xmm2, (%rax)
+; CHECK-NEXT: retq
+ %wide = load <32 x i32>, ptr %src
+ %result = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave8.v32i32(<32 x i32> %wide)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 3
+ %v4 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 4
+ %v5 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 5
+ %v6 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 6
+ %v7 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %result, 7
+ store <4 x i32> %v0, ptr %dst0
+ store <4 x i32> %v1, ptr %dst1
+ store <4 x i32> %v2, ptr %dst2
+ store <4 x i32> %v3, ptr %dst3
+ store <4 x i32> %v4, ptr %dst4
+ store <4 x i32> %v5, ptr %dst5
+ store <4 x i32> %v6, ptr %dst6
+ store <4 x i32> %v7, ptr %dst7
+ ret void
+}
>From 5f52df4fe42ac1572d820f5181b36a8e2892a6ee Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Mon, 6 Jul 2026 12:52:22 +0000
Subject: [PATCH 2/3] [LV] Emit interleave intrinsics
---
llvm/lib/IR/Intrinsics.cpp | 8 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 11 ---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 80 ++++++-------------
3 files changed, 29 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index 266af8e06a230..334838f2c3eeb 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -34,6 +34,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
@@ -1463,12 +1464,15 @@ static InterleaveIntrinsic InterleaveIntrinsics[] = {
};
Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
- assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
+ if (Factor < 2 || Factor > 8)
+ report_fatal_error("Unexpected factor for llvm.vector.interleave intrinsic");
return InterleaveIntrinsics[Factor - 2].Interleave;
}
Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
- assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
+ if (Factor < 2 || Factor > 8)
+ report_fatal_error(
+ "Unexpected factor for llvm.vector.deinterleave intrinsic");
return InterleaveIntrinsics[Factor - 2].Deinterleave;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2c23ecd67917e..f9b0c8797188e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2587,8 +2587,6 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
"Decision should not be set yet.");
auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Must have a group.");
- unsigned InterleaveFactor = Group->getFactor();
-
// If the instruction's allocated size doesn't equal its type size, it
// requires padding and will be scalarized.
auto &DL = I->getDataLayout();
@@ -2596,11 +2594,6 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
if (hasIrregularType(ScalarTy, DL))
return false;
- // For scalable vectors, the interleave factors must be <= 8 since we require
- // the (de)interleaveN intrinsics instead of shufflevectors.
- if (VF.isScalable() && InterleaveFactor > 8)
- return false;
-
// If the group involves a non-integral pointer, we may not be able to
// losslessly cast all values to a common type.
bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy);
@@ -6744,10 +6737,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
bool Result = (VF.isVector() && // Query is illegal for VF == 1
CM.getWideningDecision(IG->getInsertPos(), VF) ==
LoopVectorizationCostModel::CM_Interleave);
- // For scalable vectors, the interleave factors must be <= 8 since we
- // require the (de)interleaveN intrinsics instead of shufflevectors.
- assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
- "Unsupported interleave factor for scalable vectors");
return Result;
};
if (!getDecisionAndClampRange(ApplyIG, Range))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 68bfddea765b1..885db03ff641c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4401,20 +4401,11 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
assert(Val->getType() == VecTy && "Tried to interleave mismatched types");
#endif
- // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
- // must use intrinsics to interleave.
- if (VecTy->isScalableTy()) {
- assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");
- return Builder.CreateVectorInterleave(Vals, Name);
- }
-
- // Fixed length. Start by concatenating all vectors into a wide vector.
- Value *WideVec = concatenateVectors(Builder, Vals);
-
- // Interleave the elements into the wide vector.
- const unsigned NumElts = VecTy->getElementCount().getFixedValue();
- return Builder.CreateShuffleVector(
- WideVec, createInterleaveMask(NumElts, Factor), Name);
+ Type *DestTy = VectorType::get(VecTy->getElementType(),
+ VecTy->getElementCount() * Factor);
+ return Builder.CreateIntrinsicWithoutFolding(
+ Intrinsic::getInterleaveIntrinsicID(Factor), {DestTy}, Vals,
+ /*FMFSource=*/nullptr, Name);
}
// Try to vectorize the interleave group that \p Instr belongs to.
@@ -4428,9 +4419,11 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
// }
// To:
// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
-// %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements
-// %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements
-// %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements
+// %strided.vec = call { <4 x i32>, <4 x i32>, <4 x i32> }
+// @llvm.vector.deinterleave3.v12i32(<12 x i32> %wide.vec)
+// %R.vec = extractvalue %strided.vec, 0 ; R elements
+// %G.vec = extractvalue %strided.vec, 1 ; G elements
+// %B.vec = extractvalue %strided.vec, 2 ; B elements
//
// Or translate following interleaved store group (factor = 3):
// for (i = 0; i < N; i+=3) {
@@ -4440,10 +4433,10 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
// Pic[i+2] = B; // Member of index 2
// }
// To:
-// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
-// %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
-// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
-// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
+// %interleaved.vec = call <12 x i32>
+// @llvm.vector.interleave3.v12i32(
+// <4 x i32> %R.vec, <4 x i32> %G.vec, <4 x i32> %B.vec)
+// ; Interleave R,G,B
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
void VPInterleaveRecipe::execute(VPTransformState &State) {
assert((!needsMaskForGaps() || !State.VF.isScalable()) &&
@@ -4462,23 +4455,13 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
auto CreateGroupMask = [&BlockInMask, &State,
&InterleaveFactor](Value *MaskForGaps) -> Value * {
- if (State.VF.isScalable()) {
- assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
- assert(InterleaveFactor <= 8 &&
- "Unsupported deinterleave factor for scalable vectors");
- auto *ResBlockInMask = State.get(BlockInMask);
- SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
- return interleaveVectors(State.Builder, Ops, "interleaved.mask");
- }
-
if (!BlockInMask)
return MaskForGaps;
Value *ResBlockInMask = State.get(BlockInMask);
- Value *ShuffledMask = State.Builder.CreateShuffleVector(
- ResBlockInMask,
- createReplicatedMask(InterleaveFactor, State.VF.getFixedValue()),
- "interleaved.mask");
+ SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
+ Value *ShuffledMask =
+ interleaveVectors(State.Builder, Ops, "interleaved.mask");
return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
ShuffledMask, MaskForGaps)
: ShuffledMask;
@@ -4509,29 +4492,15 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Group->addMetadata(NewLoad);
ArrayRef<VPRecipeValue *> VPDefs = definedValues();
- if (VecTy->isScalableTy()) {
- // Scalable vectors cannot use arbitrary shufflevectors (only splats),
- // so must use intrinsics to deinterleave.
- assert(InterleaveFactor <= 8 &&
- "Unsupported deinterleave factor for scalable vectors");
- NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
- Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
- NewLoad->getType(), NewLoad,
- /*FMFSource=*/nullptr, "strided.vec");
- }
+ NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
+ Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
+ NewLoad->getType(), NewLoad,
+ /*FMFSource=*/nullptr, "strided.vec");
auto CreateStridedVector = [&InterleaveFactor, &State,
&NewLoad](unsigned Index) -> Value * {
assert(Index < InterleaveFactor && "Illegal group index");
- if (State.VF.isScalable())
- return State.Builder.CreateExtractValue(NewLoad, Index);
-
- // For fixed length VF, use shuffle to extract the sub-vectors from the
- // wide load.
- auto StrideMask =
- createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());
- return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
- "strided.vec");
+ return State.Builder.CreateExtractValue(NewLoad, Index);
};
for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
@@ -4654,8 +4623,6 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// Prepare for the vector type of the interleaved load/store.
Type *ScalarTy = getLoadStoreType(Instr);
unsigned InterleaveFactor = Group->getFactor();
- assert(InterleaveFactor <= 8 &&
- "Unsupported deinterleave/interleave factor for scalable vectors");
ElementCount WideVF = State.VF * InterleaveFactor;
auto *VecTy = VectorType::get(ScalarTy, WideVF);
@@ -4688,8 +4655,7 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
// TODO: Also manage existing metadata using VPIRMetadata.
Group->addMetadata(NewLoad);
- // Scalable vectors cannot use arbitrary shufflevectors (only splats),
- // so must use intrinsics to deinterleave.
+ // Use a deinterleave intrinsic to split the wide load into strided fields.
NewLoad = State.Builder.CreateIntrinsicWithoutFolding(
Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
NewLoad->getType(), NewLoad,
>From e9e15acb6f97bd6a35cbf5f595a26c7c5fc42d27 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Tue, 11 Aug 2026 07:10:04 +0000
Subject: [PATCH 3/3] updated tests
---
llvm/test/CodeGen/WebAssembly/interleave.ll | 4 +-
.../CodeGen/WebAssembly/memory-interleave.ll | 28 +-
.../CodeGen/WebAssembly/strided-int-mac.ll | 8 +-
.../AArch64/arbitrary-induction-step.ll | 9 +-
.../AArch64/force-target-instruction-cost.ll | 9 +-
.../LoopVectorize/AArch64/induction-costs.ll | 27 +-
...interleave-allocsize-not-equal-typesize.ll | 7 +-
.../AArch64/interleave-with-gaps.ll | 31 ++-
.../AArch64/interleave-with-runtime-checks.ll | 9 +-
...rleaved-store-of-first-order-recurrence.ll | 5 +-
.../AArch64/partial-reduce-dot-product.ll | 51 ++--
.../AArch64/partial-reduce-sub.ll | 18 +-
.../replicating-load-store-costs-apple.ll | 7 +-
.../AArch64/replicating-load-store-costs.ll | 7 +-
.../LoopVectorize/AArch64/strict-fadd.ll | 10 +-
.../AArch64/sve-tail-folding-option.ll | 18 +-
...interleave-to-widen-memory-constant-ops.ll | 16 +-
...-narrow-interleave-to-widen-memory-cost.ll | 41 ++-
...-interleave-to-widen-memory-derived-ivs.ll | 24 +-
...-interleave-to-widen-memory-multi-block.ll | 48 ++--
...eave-to-widen-memory-remove-loop-region.ll | 26 +-
...terleave-to-widen-memory-with-live-outs.ll | 25 +-
...to-widen-memory-with-wide-ops-and-casts.ll | 73 +++--
...e-to-widen-memory-with-wide-ops-chained.ll | 56 ++--
...nterleave-to-widen-memory-with-wide-ops.ll | 250 +++++++++---------
...sform-narrow-interleave-to-widen-memory.ll | 60 ++---
.../ARM/mve-gather-scatter-tailpred.ll | 3 +-
.../LoopVectorize/ARM/mve-reductions.ll | 15 +-
.../LoopVectorize/ARM/pointer_iv.ll | 18 +-
.../LoopVectorize/PowerPC/large-loop-rdx.ll | 121 +++++----
.../RISCV/interleaved-accesses.ll | 140 ++++------
.../RISCV/interleaved-store-with-gap.ll | 3 +-
.../RISCV/tail-folding-interleave.ll | 5 +-
.../LoopVectorize/X86/cast-costs.ll | 17 +-
.../X86/consecutive-ptr-uniforms.ll | 30 ++-
.../X86/cost-conditional-branches.ll | 15 +-
.../X86/fixed-order-recurrence.ll | 3 +-
.../LoopVectorize/X86/interleave-cost.ll | 23 +-
.../X86/interleave-opaque-pointers.ll | 3 +-
...terleave-ptradd-with-replicated-operand.ll | 20 +-
...rleaved-accesses-sink-store-across-load.ll | 8 +-
.../LoopVectorize/X86/interleaving.ll | 50 ++--
.../X86/limit-vf-by-tripcount.ll | 25 +-
.../LoopVectorize/X86/masked-store-cost.ll | 3 +-
.../Transforms/LoopVectorize/X86/pr47437.ll | 90 ++++---
...ond-optimization-epilogue-vectorization.ll | 12 +-
.../LoopVectorize/X86/strided_load_cost.ll | 10 +-
...-narrow-interleave-to-widen-memory-gaps.ll | 7 +-
...sform-narrow-interleave-to-widen-memory.ll | 65 +++--
.../X86/vectorize-interleaved-accesses-gap.ll | 15 +-
.../X86/vplan-native-inner-loop-only.ll | 3 +-
.../X86/vplan-single-bit-ind-var-width-4.ll | 5 +-
.../x86-interleaved-accesses-masked-group.ll | 62 +++--
...86-interleaved-store-accesses-with-gaps.ll | 6 +-
.../LoopVectorize/consecutive-ptr-uniforms.ll | 32 ++-
.../LoopVectorize/if-conversion-scalable.ll | 7 +-
.../Transforms/LoopVectorize/induction.ll | 24 +-
.../LoopVectorize/interleaved-accesses-1.ll | 3 +-
.../LoopVectorize/interleaved-accesses-2.ll | 3 +-
.../interleaved-accesses-dead-member.ll | 12 +-
...aved-accesses-different-insert-position.ll | 15 +-
.../interleaved-accesses-gep-nowrap-flags.ll | 39 +--
.../interleaved-accesses-metadata.ll | 19 +-
.../interleaved-accesses-pred-stores.ll | 12 +-
...aved-accesses-requiring-scev-predicates.ll | 6 +-
.../LoopVectorize/interleaved-accesses.ll | 112 ++++----
...28062-interleaved-accesses-narrow-group.ll | 14 +-
llvm/test/Transforms/LoopVectorize/pr31098.ll | 2 +-
.../LoopVectorize/versioning-dead-load.ll | 15 +-
.../AArch64/hoist-load-from-vector-loop.ll | 10 +-
.../PhaseOrdering/AArch64/interleave_vec.ll | 140 ++++++++--
.../AArch64/interleavevectorization.ll | 148 +++++++----
72 files changed, 1236 insertions(+), 1021 deletions(-)
diff --git a/llvm/test/CodeGen/WebAssembly/interleave.ll b/llvm/test/CodeGen/WebAssembly/interleave.ll
index c20b5e42c4850..339d385de0bd8 100644
--- a/llvm/test/CodeGen/WebAssembly/interleave.ll
+++ b/llvm/test/CodeGen/WebAssembly/interleave.ll
@@ -17,11 +17,11 @@ define hidden void @accumulate8x2(ptr dead_on_unwind noalias writable sret(%stru
; CHECK-LABEL: accumulate8x2:
; CHECK: loop
; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_u
; CHECK: i32x4.extend_low_i16x8_u
; CHECK: i32x4.add
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_u
; CHECK: i32x4.extend_low_i16x8_u
; CHECK: i32x4.add
diff --git a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
index 63cee30414a7c..d1b85bc29e6ed 100644
--- a/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
+++ b/llvm/test/CodeGen/WebAssembly/memory-interleave.ll
@@ -750,17 +750,17 @@ define hidden void @four_bytes_same_op(ptr noalias nocapture noundef writeonly %
; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.shuffle 1, 5, 9, 13, 17, 21, 25, 29, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extmul_low_i8x16_u
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+; CHECK: i8x16.shuffle 8, 24, 0, 0, 10, 26, 0, 0, 12, 28, 0, 0, 14, 30, 0, 0
; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.shuffle 2, 6, 10, 14, 18, 22, 26, 30, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.sub
; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.shuffle 3, 7, 11, 15, 19, 23, 27, 31, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i8x16.sub
-; CHECK: i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23
-; CHECK: i8x16.shuffle 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
+; CHECK: i8x16.shuffle 0, 0, 4, 20, 0, 0, 5, 21, 0, 0, 6, 22, 0, 0, 7, 23
+; CHECK: i8x16.shuffle 0, 1, 18, 19, 4, 5, 22, 23, 8, 9, 26, 27, 12, 13, 30, 31
; CHECK: v128.store
-; CHECK: i8x16.shuffle 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27
+; CHECK: i8x16.shuffle 0, 0, 0, 16, 0, 0, 1, 17, 0, 0, 2, 18, 0, 0, 3, 19
; CHECK: v128.store
define hidden void @four_bytes_split_op(ptr noalias nocapture noundef writeonly %0, ptr nocapture noundef readonly %1, ptr nocapture noundef readonly %2, i32 noundef %3) {
%5 = icmp eq i32 %3, 0
@@ -2123,21 +2123,21 @@ for.body: ; preds = %entry, %for.body
; CHECK-LABEL: two_bytes_two_floats_same_op:
; CHECK: loop
; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
; CHECK: f32x4.mul
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
@@ -2181,21 +2181,21 @@ for.body: ; preds = %entry, %for.body
; CHECK-LABEL: two_bytes_two_floats_vary_op:
; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
; CHECK: v128.load64_zero
-; CHECK: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
; CHECK: f32x4.add
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
-; CHECK: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK: i16x8.extend_low_i8x16_s
; CHECK: i32x4.extend_low_i16x8_s
; CHECK: f32x4.convert_i32x4_s
@@ -2258,7 +2258,7 @@ for.body: ; preds = %entry, %for.body
; CHECK: v128.and
; CHECK: i16x8.narrow_i32x4_u
; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
; CHECK: v128.store64_lane
define hidden void @two_floats_two_bytes_same_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
entry:
@@ -2312,7 +2312,7 @@ for.body: ; preds = %entry, %for.body
; CHECK: v128.and
; CHECK: i16x8.narrow_i32x4_u
; CHECK: i8x16.narrow_i16x8_u
-; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK: i8x16.shuffle 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
; CHECK: v128.store64_lane
define hidden void @two_floats_two_bytes_vary_op(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef writeonly captures(none) %res, i32 noundef %N) {
entry:
diff --git a/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll b/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
index d96b89f9763d8..71182dedeb9a1 100644
--- a/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
+++ b/llvm/test/CodeGen/WebAssembly/strided-int-mac.ll
@@ -1837,14 +1837,14 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: v128.load64_zero 0:p2align=0
; CHECK-NEXT: local.tee 15
; CHECK-NEXT: local.get 12
-; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 16
; CHECK-NEXT: local.get 2
; CHECK-NEXT: v128.load64_zero 0:p2align=0
; CHECK-NEXT: local.tee 17
; CHECK-NEXT: local.get 12
-; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT: i8x16.shuffle 1, 3, 5, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 18
; CHECK-NEXT: i32x4.extmul_low_i16x8_s
@@ -1854,7 +1854,7 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: local.get 16
; CHECK-NEXT: local.get 17
; CHECK-NEXT: local.get 12
-; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 17
; CHECK-NEXT: i32x4.extmul_low_i16x8_s
@@ -1864,7 +1864,7 @@ define hidden { i32, i32, i32, i32 } @bb41_inner_loop(ptr nocapture %lhs, ptr no
; CHECK-NEXT: local.get 18
; CHECK-NEXT: local.get 15
; CHECK-NEXT: local.get 12
-; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT: i8x16.shuffle 0, 2, 4, 6, 8, 10, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0
; CHECK-NEXT: i16x8.extend_low_i8x16_s
; CHECK-NEXT: local.tee 15
; CHECK-NEXT: i32x4.extmul_low_i16x8_s
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
index 66b800c850892..637a3a66c6908 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
@@ -103,11 +103,9 @@ for.end:
; CHECK-LABEL: @ptr_ind_plus2(
; CHECK: %[[V0:.*]] = load <8 x i32>
-; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %[[V0]])
; CHECK: %[[V1:.*]] = load <8 x i32>
-; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %[[V1]])
; CHECK: mul nsw <4 x i32>
; CHECK: mul nsw <4 x i32>
; CHECK: add <4 x i32>
@@ -117,8 +115,7 @@ for.end:
; FORCE-VEC-LABEL: @ptr_ind_plus2(
; FORCE-VEC: %[[V:.*]] = load <4 x i32>
-; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
-; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; FORCE-VEC: call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %[[V]])
; FORCE-VEC: mul nsw <2 x i32>
; FORCE-VEC: add <2 x i32>
; FORCE-VEC: %index.next = add nuw i64 %index, 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 4bc2adc5af8fc..4525c3aabe10b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -449,8 +449,10 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST1-NEXT: store <48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST1-NEXT: store <48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -516,7 +518,8 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST10-NEXT: store <48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index aef0099f875d8..d0d2375ff6d70 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -640,18 +640,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP79:%.*]] = load double, ptr [[TMP65]], align 8, !alias.scope [[META17]]
; CHECK-NEXT: [[TMP80:%.*]] = insertelement <2 x double> poison, double [[TMP78]], i64 0
; CHECK-NEXT: [[TMP81:%.*]] = insertelement <2 x double> [[TMP80]], double [[TMP79]], i64 1
-; CHECK-NEXT: [[TMP82:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP69]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP82]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> [[TMP69]])
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP38]], align 8, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; CHECK-NEXT: [[TMP83:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP73]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC17:%.*]] = shufflevector <4 x double> [[TMP83]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP39]], align 8, !alias.scope [[META20]], !noalias [[META17]]
-; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP77]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC18:%.*]] = shufflevector <4 x double> [[TMP84]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC18]], ptr [[TMP40]], align 8, !alias.scope [[META20]], !noalias [[META17]]
-; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP81]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC19:%.*]] = shufflevector <4 x double> [[TMP85]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC19]], ptr [[TMP41]], align 8, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> [[TMP73]])
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP39]], align 8, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-NEXT: [[INTERLEAVED_VEC12:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> [[TMP77]])
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC12]], ptr [[TMP40]], align 8, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-NEXT: [[INTERLEAVED_VEC13:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> [[TMP81]])
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC13]], ptr [[TMP41]], align 8, !alias.scope [[META20]], !noalias [[META17]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -691,9 +687,8 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP101:%.*]] = load double, ptr [[TMP99]], align 8, !alias.scope [[META17]]
; CHECK-NEXT: [[TMP102:%.*]] = insertelement <2 x double> poison, double [[TMP100]], i64 0
; CHECK-NEXT: [[TMP103:%.*]] = insertelement <2 x double> [[TMP102]], double [[TMP101]], i64 1
-; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP103]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC26:%.*]] = shufflevector <4 x double> [[TMP104]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC26]], ptr [[TMP93]], align 8, !alias.scope [[META20]], !noalias [[META17]]
+; CHECK-NEXT: [[INTERLEAVED_VEC19:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> [[TMP103]])
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC19]], ptr [[TMP93]], align 8, !alias.scope [[META20]], !noalias [[META17]]
; CHECK-NEXT: [[INDEX_NEXT27]] = add nuw i64 [[INDEX24]], 2
; CHECK-NEXT: [[VEC_IND_NEXT28]] = add <2 x i32> [[VEC_IND25]], splat (i32 4)
; CHECK-NEXT: [[TMP105:%.*]] = icmp eq i64 [[INDEX_NEXT27]], [[N_VEC23]]
@@ -941,7 +936,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[TMP131:%.*]] = insertelement <8 x ptr> [[TMP130]], ptr [[TMP123]], i64 7
; CHECK-NEXT: [[INDEX_NEXT28]] = add nuw i64 [[INDEX19]], 8
; CHECK-NEXT: [[TMP132:%.*]] = icmp eq i64 [[INDEX_NEXT28]], [[N_VEC18]]
-; CHECK-NEXT: br i1 [[TMP132]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP132]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP133:%.*]] = ptrtoint <8 x ptr> [[TMP131]] to <8 x i64>
; CHECK-NEXT: [[TMP134:%.*]] = extractelement <8 x i64> [[TMP133]], i64 7
@@ -961,7 +956,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[PTR_INT:%.*]] = ptrtoint ptr [[PTR_IV_NEXT]] to i64
; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[IV]], [[COUNT]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[PTR_INT_LCSSA:%.*]] = phi i64 [ [[PTR_INT]], %[[LOOP]] ], [ [[TMP87]], %[[MIDDLE_BLOCK]] ], [ [[TMP134]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[PTR_INT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
index f1b732f9d67cd..2a5b7333de73e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll
@@ -28,13 +28,14 @@ define void @pr58722_load_interleave_group(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = load i24, ptr [[TMP9]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i24, ptr [[TMP9]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP14:%.*]] = load i24, ptr [[TMP10]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP15:%.*]] = load i24, ptr [[TMP11]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP16:%.*]] = load i24, ptr [[TMP12]], align 4, !alias.scope [[META0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 2733930833e7e..b31cb40184f0d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -23,7 +23,8 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP1:%.*]] = getelementptr { [4 x i8] }, ptr [[SRC]], i64 [[INDEX]], i32 0, i64 3
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
-; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i8> [[WIDE_VEC]], <64 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave4.v64i8(<64 x i8> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
; CHECK-NOTF-NEXT: [[TMP3:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP2]])
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP3]])
@@ -128,7 +129,8 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NOTF-NEXT: [[TMP1:%.*]] = getelementptr { [4 x i8] }, ptr [[SRC]], i64 [[INDEX]], i32 0, i64 3
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <64 x i8>, ptr [[TMP1]], align 1
-; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i8> [[WIDE_VEC]], <64 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave4.v64i8(<64 x i8> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
; CHECK-NOTF-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
; CHECK-NOTF-NEXT: [[TMP3:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP2]])
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = call <16 x i32> @llvm.umin.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP3]])
@@ -231,7 +233,8 @@ define void @main_vector_loop_fixed_single_vector_iteration_with_runtime_checks(
; CHECK-NOTF-NEXT: [[TMP3:%.*]] = add i64 [[TMP0]], 6
; CHECK-NOTF-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[J]], i64 [[TMP0]]
; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC1]], 0
; CHECK-NOTF-NEXT: [[TMP5:%.*]] = trunc <4 x i64> [[STRIDED_VEC]] to <4 x i16>
; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP0]]
; CHECK-NOTF-NEXT: [[TMP11:%.*]] = getelementptr i16, ptr [[K]], i64 [[TMP1]]
@@ -463,9 +466,10 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP45]], %[[SCALAR_PH]] ], [ [[TMP49:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC17:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC17]])
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 0
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 1
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 3
; CHECK-NOTF-NEXT: [[TMP47:%.*]] = add <4 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
; CHECK-NOTF-NEXT: [[TMP48:%.*]] = add <4 x i32> [[TMP47]], [[STRIDED_VEC19]]
; CHECK-NOTF-NEXT: [[TMP49]] = add <4 x i32> [[TMP48]], [[STRIDED_VEC20]]
@@ -478,11 +482,11 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br i1 [[CMP_N22]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP44]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX20:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP44]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
; CHECK-NOTF: [[LOOP1]]:
; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX22]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX20]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 0
; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[LD1]]
@@ -775,9 +779,10 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP49]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
; CHECK-NOTF-NEXT: [[WIDE_VEC17:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <16 x i32> [[WIDE_VEC17]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NOTF-NEXT: [[STRIDED_VEC17:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC17]])
+; CHECK-NOTF-NEXT: [[STRIDED_VEC18:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 0
+; CHECK-NOTF-NEXT: [[STRIDED_VEC19:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 1
+; CHECK-NOTF-NEXT: [[STRIDED_VEC20:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 2
; CHECK-NOTF-NEXT: [[TMP51:%.*]] = add <4 x i32> [[VEC_PHI16]], [[STRIDED_VEC18]]
; CHECK-NOTF-NEXT: [[TMP52:%.*]] = add <4 x i32> [[TMP51]], [[STRIDED_VEC19]]
; CHECK-NOTF-NEXT: [[TMP53]] = add <4 x i32> [[TMP52]], [[STRIDED_VEC20]]
@@ -789,11 +794,11 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
; CHECK-NOTF-NEXT: br label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-NOTF: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NOTF-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP46]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX19:%.*]] = phi i32 [ [[TMP55]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP46]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NOTF-NEXT: br label %[[LOOP1:.*]]
; CHECK-NOTF: [[LOOP1]]:
; CHECK-NOTF-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NOTF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX19]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV1]], i32 0
; CHECK-NOTF-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
; CHECK-NOTF-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[LD1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
index 9f21ec1806703..b93884a38651c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
@@ -39,8 +39,10 @@ define void @interleave_groups_separated_by_offset(ptr %A, i64 %offset) {
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 32
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: store <32 x i8> zeroinitializer, ptr [[NEXT_GEP11]], align 1
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; CHECK-NEXT: store <32 x i8> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT: [[INTERLEAVED_VEC12:%.*]] = call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; CHECK-NEXT: store <32 x i8> [[INTERLEAVED_VEC12]], ptr [[NEXT_GEP11]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -57,7 +59,8 @@ define void @interleave_groups_separated_by_offset(ptr %A, i64 %offset) {
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX12]], 1
; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP13]], align 1
+; CHECK-NEXT: [[INTERLEAVED_VEC17:%.*]] = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> zeroinitializer, <8 x i8> zeroinitializer)
+; CHECK-NEXT: store <16 x i8> [[INTERLEAVED_VEC17]], ptr [[NEXT_GEP13]], align 1
; CHECK-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX12]], 8
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT14]], 992
; CHECK-NEXT: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
index f2af293f365d3..e56d77363e7a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
@@ -13,10 +13,7 @@ define void @interleaved_store_first_order_recurrence(ptr noalias %src, ptr %dst
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> zeroinitializer, <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i32> [[TMP11]], <12 x i32> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> zeroinitializer, <4 x i32> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT]])
; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index a9e8233615783..0de04f73c944b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -2329,14 +2329,15 @@ define void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #
; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = shl nsw i64 [[INDEX]], 3
; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP9]]
; CHECK-INTERLEAVE1-NEXT: [[WIDE_VEC:%.*]] = load <128 x i8>, ptr [[TMP10]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC13:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126>
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave8.v128i8(<128 x i8> [[WIDE_VEC]])
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 1
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 2
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 3
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC11:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 4
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC12:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 5
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC13:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 6
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC14:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 7
; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = sext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = mul nsw <16 x i32> [[TMP11]], [[TMP12]]
@@ -2429,14 +2430,15 @@ define void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = shl nsw i64 [[INDEX]], 3
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP9]]
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC:%.*]] = load <128 x i8>, ptr [[TMP10]], align 1
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC13:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126>
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave8.v128i8(<128 x i8> [[WIDE_VEC]])
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 1
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 2
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 3
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC11:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 4
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC12:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 5
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC13:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 6
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC14:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 7
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = sext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = mul nsw <16 x i32> [[TMP11]], [[TMP12]]
@@ -2529,14 +2531,15 @@ define void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #
; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = shl nsw i64 [[INDEX]], 3
; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP9]]
; CHECK-MAXBW-NEXT: [[WIDE_VEC:%.*]] = load <128 x i8>, ptr [[TMP10]], align 1
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 64, i32 72, i32 80, i32 88, i32 96, i32 104, i32 112, i32 120>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 65, i32 73, i32 81, i32 89, i32 97, i32 105, i32 113, i32 121>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 66, i32 74, i32 82, i32 90, i32 98, i32 106, i32 114, i32 122>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 67, i32 75, i32 83, i32 91, i32 99, i32 107, i32 115, i32 123>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 68, i32 76, i32 84, i32 92, i32 100, i32 108, i32 116, i32 124>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 69, i32 77, i32 85, i32 93, i32 101, i32 109, i32 117, i32 125>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC13:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 70, i32 78, i32 86, i32 94, i32 102, i32 110, i32 118, i32 126>
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave8.v128i8(<128 x i8> [[WIDE_VEC]])
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 1
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 2
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 3
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC11:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 4
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC12:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 5
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC13:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 6
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC14:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 7
; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = sext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = mul nsw <16 x i32> [[TMP11]], [[TMP12]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index 83e50bb32136e..94fe28103f807 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -141,7 +141,8 @@ define i64 @partial_reduce_sub_sext_mul(ptr %x) #1 {
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP0]]
; CHECK-INTERLEAVE1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[STRIDED_VEC]] to <4 x i64>
@@ -182,13 +183,17 @@ define i64 @partial_reduce_sub_sext_mul(ptr %x) #1 {
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP23]]
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP6]]
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC6:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC2]])
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC6]], 0
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC6:%.*]] = load <8 x i32>, ptr [[TMP24]], align 4
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x i32> [[WIDE_VEC6]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC8:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC6]])
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC8]], 0
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC8:%.*]] = load <8 x i32>, ptr [[TMP25]], align 4
-; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC3]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC9:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC8]])
+; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC3]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC9]], 0
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> [[STRIDED_VEC5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC5]], <4 x i32> [[STRIDED_VEC7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -235,7 +240,8 @@ define i64 @partial_reduce_sub_sext_mul(ptr %x) #1 {
; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP0]]
; CHECK-MAXBW-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; CHECK-MAXBW-NEXT: [[STRIDED_VEC]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-MAXBW-NEXT: [[STRIDED_VEC]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[STRIDED_VEC]] to <4 x i64>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index c62f6ef0acc4d..a64817bc502c8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -569,9 +569,10 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[SRC_0]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave3.v6f64(<6 x double> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 1
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 2
; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[STRIDED_VEC]], splat (double 3.000000e+00)
; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[STRIDED_VEC1]], splat (double 3.000000e+00)
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index 325c3ffe4eac7..914da55760e29 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -565,9 +565,10 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[SRC_0]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave3.v6f64(<6 x double> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 1
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC3]], 2
; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[STRIDED_VEC]], splat (double 3.000000e+00)
; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[STRIDED_VEC1]], splat (double 3.000000e+00)
; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
index 6ca7c01bdf73b..3f5702f5e3eba 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
@@ -261,8 +261,9 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
; CHECK-ORDERED: %[[VEC_PHI1:.*]] = phi float [ %[[LOAD2]], %vector.ph ], [ %[[RDX2:.*]], %vector.body ]
; CHECK-ORDERED: %[[VEC_PHI2:.*]] = phi float [ %[[LOAD1]], %vector.ph ], [ %[[RDX1:.*]], %vector.body ]
; CHECK-ORDERED: %[[WIDE_LOAD:.*]] = load <8 x float>, ptr
-; CHECK-ORDERED: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-ORDERED: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-ORDERED: %[[DEINT:.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[WIDE_LOAD]])
+; CHECK-ORDERED: %[[STRIDED1:.*]] = extractvalue { <4 x float>, <4 x float> } %[[DEINT]], 0
+; CHECK-ORDERED: %[[STRIDED2:.*]] = extractvalue { <4 x float>, <4 x float> } %[[DEINT]], 1
; CHECK-ORDERED: %[[RDX2]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[STRIDED2]])
; CHECK-ORDERED: %[[RDX1]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI2]], <4 x float> %[[STRIDED1]])
; CHECK-ORDERED: for.end
@@ -279,8 +280,9 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi <4 x float> [ %[[INS2]], %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]
; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi <4 x float> [ %[[INS1]], %vector.ph ], [ %[[VEC_FADD1:.*]], %vector.body ]
; CHECK-UNORDERED: %[[WIDE_LOAD:.*]] = load <8 x float>, ptr
-; CHECK-UNORDERED: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-UNORDERED: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-UNORDERED: %[[DEINT:.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[WIDE_LOAD]])
+; CHECK-UNORDERED: %[[STRIDED1:.*]] = extractvalue { <4 x float>, <4 x float> } %[[DEINT]], 0
+; CHECK-UNORDERED: %[[STRIDED2:.*]] = extractvalue { <4 x float>, <4 x float> } %[[DEINT]], 1
; CHECK-UNORDERED: %[[VEC_FADD1]] = fadd <4 x float> %[[STRIDED1:.*]], %[[VEC_PHI1]]
; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[STRIDED2:.*]], %[[VEC_PHI2]]
; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index e80df2a09824c..e3e6843d993d8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -284,38 +284,32 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
; CHECK-NOTF-LABEL: @interleave(
; CHECK-NOTF: vector.body:
; CHECK-NOTF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
; CHECK-TF-LABEL: @interleave(
; CHECK-TF: vector.body:
; CHECK-TF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
; CHECK-TF-NORED-LABEL: @interleave(
; CHECK-TF-NORED: vector.body:
; CHECK-TF-NORED: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NORED: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
; CHECK-TF-NOREC-LABEL: @interleave(
; CHECK-TF-NOREC: vector.body:
; CHECK-TF-NOREC: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREC: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
; CHECK-TF-NOREV-LABEL: @interleave(
; CHECK-TF-NOREV: vector.body:
; CHECK-TF-NOREV: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREV: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
; CHECK-NEOVERSE-V1-LABEL: @interleave(
; CHECK-NEOVERSE-V1: vector.body:
; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1: %{{.*}} = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %[[LOAD]])
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
index 4904f26de1558..42ed09ef9f387 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
@@ -278,22 +278,22 @@ define void @test_add_double_same_var_args_at_different_positions(ptr %res, ptr
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC5]], 1
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP3]], <2 x double> [[TMP5]])
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP4]], <2 x double> [[TMP6]])
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index fbf352c2ff56f..b52a1cd5106fe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -22,28 +22,30 @@ define void @test_complex_add_float(ptr %res, ptr noalias %A, ptr noalias %B, i6
; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[TMP1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[GEP_A_0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC5]], 1
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[WIDE_VEC5:%.*]] = load <8 x float>, ptr [[GEP_B_0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x float> [[WIDE_VEC5]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC8:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC5]])
+; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC8]], 0
+; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC8]], 1
; CHECK-NEXT: [[WIDE_VEC8:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x float> [[WIDE_VEC8]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC11:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC8]])
+; CHECK-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC11]], 0
+; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC11]], 1
; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x float> [[STRIDED_VEC]], [[STRIDED_VEC6]]
; CHECK-NEXT: [[TMP7:%.*]] = fadd <4 x float> [[STRIDED_VEC3]], [[STRIDED_VEC9]]
; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x float> [[STRIDED_VEC1]], [[STRIDED_VEC7]]
; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x float> [[STRIDED_VEC4]], [[STRIDED_VEC10]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP6]], <4 x float> [[TMP8]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <8 x float> [[TMP13]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP7]], <4 x float> [[TMP9]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC11]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -190,17 +192,13 @@ define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %d
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP10]]
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP11]]
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <2 x double> [[TMP18]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP18]], <2 x double> zeroinitializer)
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP22]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC9:%.*]] = shufflevector <4 x double> [[TMP27]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC9:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP19]], <2 x double> zeroinitializer)
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC9]], ptr [[TMP23]], align 8
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC10:%.*]] = shufflevector <4 x double> [[TMP28]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC10:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP20]], <2 x double> zeroinitializer)
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC10]], ptr [[TMP24]], align 8
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <4 x double> [[TMP29]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP21]], <2 x double> zeroinitializer)
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP25]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -222,8 +220,7 @@ define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %d
; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <2 x double>, ptr [[TMP31]], align 8
; CHECK-NEXT: [[TMP33:%.*]] = fmul <2 x double> [[WIDE_LOAD15]], splat (double 5.000000e+00)
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[INDEX14]]
-; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC16:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP33]], <2 x double> zeroinitializer)
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
; CHECK-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[INDEX14]], 2
; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC13]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
index a39f080dc7241..8192f446c21ac 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
@@ -91,11 +91,11 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF4-NEXT: [[OFFSET_IDX:%.*]] = add nuw i64 16, [[TMP5]]
; VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP6]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP11:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP10:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP11]], <4 x double> [[TMP10]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -221,10 +221,10 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF4-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; VF4-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[NEXT_GEP]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP13:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP13:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP12:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 1
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP13]], <4 x double> [[TMP12]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP6]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -314,12 +314,12 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF4-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
; VF4-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP3]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 478a724f49c21..f58a62da2713e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -17,10 +17,10 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF2IC1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
-; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC1-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2IC1-NEXT: [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2IC1-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
+; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP5]])
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF2IC1: [[PRED_STORE_IF]]:
@@ -56,16 +56,16 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP2]]
; VF2IC2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; VF2IC2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2IC2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2IC2-NEXT: [[TMP9:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2IC2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
-; VF2IC2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x i64> [[WIDE_VEC2]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x i64> [[WIDE_VEC2]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
-; VF2IC2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC2]])
+; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2IC2-NEXT: [[TMP8:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
+; VF2IC2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP9]], <2 x i64> [[TMP10]])
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP11]], <2 x i64> [[TMP8]])
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF2IC2: [[PRED_STORE_IF]]:
@@ -141,10 +141,10 @@ define void @load_store_interleave_group_block_var_cond(ptr noalias %data, ptr %
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF2IC1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
-; VF2IC1-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC1-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2IC1-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2IC1-NEXT: [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
+; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[MASKS]], i64 [[INDEX]]
; VF2IC1-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
@@ -185,16 +185,16 @@ define void @load_store_interleave_group_block_var_cond(ptr noalias %data, ptr %
; VF2IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2IC2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP2]]
; VF2IC2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; VF2IC2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2IC2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2IC2-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2IC2-NEXT: [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2IC2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
-; VF2IC2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x i64> [[WIDE_VEC2]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2IC2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x i64> [[WIDE_VEC2]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
-; VF2IC2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC2]])
+; VF2IC2-NEXT: [[TMP12:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2IC2-NEXT: [[TMP13:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
+; VF2IC2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
-; VF2IC2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP12]], <2 x i64> [[TMP13]])
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
; VF2IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[MASKS]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
index 73fd53011e956..7af3fef5f2cb6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
@@ -189,16 +189,17 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC5:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC5]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC5]], 1
; VF2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
-; VF2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x float> [[WIDE_VEC2]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[WIDE_VEC2]])
+; VF2-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[STRIDED_VEC]], [[STRIDED_VEC3]]
; VF2-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[STRIDED_VEC1]], [[STRIDED_VEC4]]
; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[RES]], i64 [[INDEX]]
-; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]])
; VF2-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
@@ -216,15 +217,16 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[A]], align 4
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC5]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC5]], 1
; VF4-NEXT: [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[B]], align 4
-; VF4-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC2]])
+; VF4-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[STRIDED_VEC]], [[STRIDED_VEC3]]
; VF4-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[STRIDED_VEC1]], [[STRIDED_VEC4]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP2]], <4 x float> [[TMP3]])
; VF4-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[RES]], align 4
; VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; VF4: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
index 6fd49e2e99f18..addff97af9240 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll
@@ -32,7 +32,8 @@ define i64 @test_wide_op_live_out_constant_store_group(ptr noalias %res, ptr noa
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x i64> zeroinitializer, ptr [[TMP0]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> zeroinitializer, <4 x i64> zeroinitializer)
+; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP0]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; VF4-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -71,11 +72,11 @@ define i64 @test_wide_op_live_out(ptr noalias %res, ptr noalias %A) {
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC1]], 0
+; VF2-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC1]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[STRIDED_VEC]], <2 x i64> [[TMP5]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -97,11 +98,11 @@ define i64 @test_wide_op_live_out(ptr noalias %res, ptr noalias %A) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[A]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC1]], 0
+; VF4-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC1]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[STRIDED_VEC]], <4 x i64> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[STRIDED_VEC]], <4 x i64> [[TMP2]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -147,7 +148,8 @@ define i64 @test_wide_load_live_out_constant_store_group(ptr noalias %res, ptr n
; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF2-NEXT: store <4 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> zeroinitializer, <2 x i64> zeroinitializer)
+; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -168,7 +170,8 @@ define i64 @test_wide_load_live_out_constant_store_group(ptr noalias %res, ptr n
; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { i64, i64 }, ptr [[RES]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> zeroinitializer, <4 x i64> zeroinitializer)
+; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
index da6198034970d..7dca43afa5bcb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll
@@ -43,8 +43,7 @@ define void @test_2xi64_matching_zext_interleave_group(ptr noalias %dst, ptr %sr
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP2]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -115,8 +114,7 @@ define void @test_2xi64_matching_sext_interleave_group(ptr noalias %dst, ptr %sr
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP2]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -164,8 +162,7 @@ define void @test_2xi64_mismatching_cast_interleave_group(ptr noalias %dst, ptr
; VF2-NEXT: [[TMP2:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP4:%.*]] = sext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
-; VF2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP4]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -189,8 +186,7 @@ define void @test_2xi64_mismatching_cast_interleave_group(ptr noalias %dst, ptr
; VF4-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -263,8 +259,7 @@ define void @test_2xi64_matching_cast_add_interleave_group(ptr noalias %dst, ptr
; VF4-NEXT: [[TMP2:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP3:%.*]] = add <4 x i64> [[TMP2]], splat (i64 2)
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP3]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -316,8 +311,7 @@ define void @test_2xi64_mismatching_cast_add_interleave_group(ptr noalias %dst,
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP5:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; VF2-NEXT: [[TMP6:%.*]] = add <2 x i64> [[TMP5]], splat (i64 2)
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -343,8 +337,7 @@ define void @test_2xi64_mismatching_cast_add_interleave_group(ptr noalias %dst,
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP5:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
; VF4-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], splat (i64 2)
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP6]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -419,8 +412,7 @@ define void @test_2xi64_add_cast_interleave_group(ptr noalias %dst, ptr %src) {
; VF4-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 2)
; VF4-NEXT: [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP3]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -472,8 +464,7 @@ define void @test_2xi64_mismatching_add_cast_interleave_group(ptr noalias %dst,
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP5:%.*]] = sub <2 x i32> [[WIDE_LOAD]], splat (i32 2)
; VF2-NEXT: [[TMP6:%.*]] = zext <2 x i32> [[TMP5]] to <2 x i64>
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -499,8 +490,7 @@ define void @test_2xi64_mismatching_add_cast_interleave_group(ptr noalias %dst,
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP5:%.*]] = sub <4 x i32> [[WIDE_LOAD]], splat (i32 2)
; VF4-NEXT: [[TMP6:%.*]] = zext <4 x i32> [[TMP5]] to <4 x i64>
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP6]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -551,8 +541,7 @@ define void @test_2xi64_add_mismatching_cast_interleave_group(ptr noalias %dst,
; VF2-NEXT: [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP5:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
-; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP5]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -577,8 +566,7 @@ define void @test_2xi64_add_mismatching_cast_interleave_group(ptr noalias %dst,
; VF4-NEXT: [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP5:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
-; VF4-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP5]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -630,8 +618,7 @@ define void @test_2xi64_sub_mismatching_ops_cast_interleave_group(ptr noalias %d
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP5:%.*]] = sub <2 x i32> splat (i32 2), [[WIDE_LOAD]]
; VF2-NEXT: [[TMP6:%.*]] = zext <2 x i32> [[TMP5]] to <2 x i64>
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -657,8 +644,7 @@ define void @test_2xi64_sub_mismatching_ops_cast_interleave_group(ptr noalias %d
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP5:%.*]] = sub <4 x i32> splat (i32 2), [[WIDE_LOAD]]
; VF4-NEXT: [[TMP6:%.*]] = zext <4 x i32> [[TMP5]] to <4 x i64>
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP6]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -705,15 +691,15 @@ define void @test_2xdouble_mismatching_trunc_result_types(ptr noalias %dst, ptr
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP2:%.*]] = trunc <2 x i64> [[STRIDED_VEC]] to <2 x i32>
; VF2-NEXT: [[TMP3:%.*]] = sitofp <2 x i32> [[TMP2]] to <2 x double>
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP0]]
; VF2-NEXT: [[TMP5:%.*]] = trunc <2 x i64> [[STRIDED_VEC1]] to <2 x i16>
; VF2-NEXT: [[TMP6:%.*]] = sitofp <2 x i16> [[TMP5]] to <2 x double>
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP3]], <2 x double> [[TMP6]])
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -734,15 +720,15 @@ define void @test_2xdouble_mismatching_trunc_result_types(ptr noalias %dst, ptr
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[STRIDED_VEC]] to <4 x i32>
; VF4-NEXT: [[TMP3:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x double>
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP5:%.*]] = trunc <4 x i64> [[STRIDED_VEC1]] to <4 x i16>
; VF4-NEXT: [[TMP6:%.*]] = sitofp <4 x i16> [[TMP5]] to <4 x double>
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP3]], <4 x double> [[TMP6]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -793,7 +779,8 @@ define void @test_2xdouble_mismatching_sitofp_operand_types(ptr noalias %dst, pt
; VF2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP3]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC1]], 0
; VF2-NEXT: [[TMP4:%.*]] = sitofp <2 x i32> [[STRIDED_VEC]] to <2 x double>
; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP1]]
; VF2-NEXT: [[TMP6:%.*]] = or disjoint i64 [[TMP1]], 1
@@ -805,8 +792,7 @@ define void @test_2xdouble_mismatching_sitofp_operand_types(ptr noalias %dst, pt
; VF2-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> poison, i16 [[TMP10]], i64 0
; VF2-NEXT: [[TMP13:%.*]] = insertelement <2 x i16> [[TMP12]], i16 [[TMP11]], i64 1
; VF2-NEXT: [[TMP14:%.*]] = sitofp <2 x i16> [[TMP13]] to <2 x double>
-; VF2-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP4]], <2 x double> [[TMP14]])
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
@@ -826,16 +812,17 @@ define void @test_2xdouble_mismatching_sitofp_operand_types(ptr noalias %dst, pt
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; VF4-NEXT: [[TMP2:%.*]] = sitofp <4 x i32> [[STRIDED_VEC]] to <4 x double>
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP4:%.*]] = or disjoint i64 [[TMP0]], 1
; VF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP4]]
; VF4-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i16>, ptr [[TMP5]], align 8
-; VF4-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i16> [[WIDE_VEC1]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC1]])
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC3]], 0
; VF4-NEXT: [[TMP6:%.*]] = sitofp <4 x i16> [[STRIDED_VEC2]] to <4 x double>
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP2]], <4 x double> [[TMP6]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 60b13e460d8c9..8c3a8be8938b9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -72,14 +72,14 @@ define void @test_2xi64_mixed_opcodes1(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = add <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP6:%.*]] = xor <2 x i64> [[TMP5]], splat (i64 2)
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -130,14 +130,14 @@ define void @test_2xi64_mixed_opcodes2(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = xor <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = add <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP6:%.*]] = add <2 x i64> [[TMP5]], splat (i64 2)
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -301,12 +301,12 @@ define void @test_2xi64_mul_sub_mismatched_ops2(ptr noalias %data, ptr noalias %
; VF2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP9]]
; VF2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 -8
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP12:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], splat (i64 2)
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -357,14 +357,14 @@ define void @test_2xi64_mul_sub_mismatched_op_order(ptr noalias %data, ptr noali
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> splat (i64 2), [[TMP5]]
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -472,16 +472,16 @@ define void @test_2xi64_mul_add_xor_mismatched_opcodes1(ptr noalias %data, ptr n
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = add <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP5:%.*]] = xor <2 x i64> splat (i64 4), [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP7:%.*]] = add <2 x i64> [[TMP6]], splat (i64 2)
; VF2-NEXT: [[TMP8:%.*]] = xor <2 x i64> splat (i64 4), [[TMP7]]
-; VF2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP8]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -534,16 +534,16 @@ define void @test_2xi64_mul_add_xor_mismatched_opcodes2(ptr noalias %data, ptr n
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = add <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP5:%.*]] = xor <2 x i64> splat (i64 4), [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP7:%.*]] = shl <2 x i64> [[TMP6]], splat (i64 1)
; VF2-NEXT: [[TMP8:%.*]] = xor <2 x i64> splat (i64 4), [[TMP7]]
-; VF2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP8]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -602,13 +602,13 @@ define void @test_2xi64_mul_add_xor_mismatched_ops(ptr noalias %data, ptr noalia
; VF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]]
; VF2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 -8
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP9:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]
; VF2-NEXT: [[TMP7:%.*]] = add <2 x i64> [[TMP6]], splat (i64 2)
; VF2-NEXT: [[TMP8:%.*]] = xor <2 x i64> splat (i64 4), [[TMP7]]
-; VF2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP8]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 29849f2c3d008..1c208fea5c3a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -38,12 +38,12 @@ define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr no
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP1]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP3:%.*]] = fneg <4 x double> [[STRIDED_VEC]]
; VF4-NEXT: [[TMP4:%.*]] = fneg <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP3]], <4 x double> [[TMP4]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -93,8 +93,7 @@ define void @test_2xi64_unary_op_wide_load(ptr noalias %data, ptr noalias %A, pt
; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP14]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
; VF2-NEXT: [[TMP9:%.*]] = fneg <2 x double> [[WIDE_LOAD1]]
-; VF2-NEXT: [[TMP17:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC4:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP15]], <2 x double> [[TMP9]])
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC4]], ptr [[TMP20]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP14]], 2
; VF2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -120,8 +119,7 @@ define void @test_2xi64_unary_op_wide_load(ptr noalias %data, ptr noalias %A, pt
; VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; VF4-NEXT: [[TMP8:%.*]] = fneg <4 x double> [[WIDE_LOAD1]]
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP9]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP4]], <4 x double> [[TMP8]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -195,12 +193,12 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor) {
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP11]], align 8
-; VF4-NEXT: [[TMP19:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[TMP41:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP41:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP20:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP19]]
; VF4-NEXT: [[TMP42:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP41]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP20]], <4 x i64> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP20]], <4 x i64> [[TMP42]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -249,12 +247,12 @@ define void @test_2xi64_different_opcodes(ptr noalias %data, ptr noalias %factor
; VF2-NEXT: [[TMP3:%.*]] = shl nsw i64 [[TMP0]], 1
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF2-NEXT: [[TMP6:%.*]] = add <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -277,12 +275,12 @@ define void @test_2xi64_different_opcodes(ptr noalias %data, ptr noalias %factor
; VF4-NEXT: [[TMP3:%.*]] = shl nsw i64 [[TMP0]], 1
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; VF4-NEXT: [[TMP6:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP6]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -331,12 +329,12 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no
; VF2-NEXT: [[TMP15:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP15]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP16]], align 8
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[TMP20:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[TMP20:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2-NEXT: [[TMP21:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP20]]
; VF2-NEXT: [[TMP24:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP11]]
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP21]], <2 x i64> [[TMP24]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP21]], <2 x i64> [[TMP24]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP16]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -359,12 +357,12 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no
; VF4-NEXT: [[TMP27:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP27]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP28]], align 8
-; VF4-NEXT: [[TMP19:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[TMP36:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP36:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP37:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP36]]
; VF4-NEXT: [[TMP42:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP19]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP37]], <4 x i64> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP37]], <4 x i64> [[TMP42]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP28]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -413,12 +411,12 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa
; VF2-NEXT: [[TMP6:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP7]], align 8
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[TMP21:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[TMP21:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP11]]
; VF2-NEXT: [[TMP22:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP21]]
-; VF2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP22]], <2 x i64> [[TMP12]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -441,12 +439,12 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP11]], align 8
-; VF4-NEXT: [[TMP19:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[TMP37:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP37:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP20:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP19]]
; VF4-NEXT: [[TMP38:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP37]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP38]], <4 x i64> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP38]], <4 x i64> [[TMP20]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -495,12 +493,12 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa
; VF2-NEXT: [[TMP6:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP7]], align 8
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[TMP21:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[TMP21:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
; VF2-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP11]]
; VF2-NEXT: [[TMP22:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP21]]
-; VF2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP22]], <2 x i64> [[TMP12]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -523,12 +521,12 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP11]], align 8
-; VF4-NEXT: [[TMP19:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[TMP37:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP37:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP20:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP19]]
; VF4-NEXT: [[TMP38:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP37]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP38]], <4 x i64> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP38]], <4 x i64> [[TMP20]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -581,12 +579,12 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF2-NEXT: [[TMP14:%.*]] = or disjoint i64 [[TMP6]], 1
; VF2-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP15]], align 8
-; VF2-NEXT: [[TMP19:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP19:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
; VF2-NEXT: [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP20]], align 8
; VF2-NEXT: [[TMP22:%.*]] = mul <2 x i64> [[WIDE_LOAD2]], [[TMP19]]
-; VF2-NEXT: [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP22]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP13]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP8]], <2 x i64> [[TMP22]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
@@ -612,12 +610,12 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF4-NEXT: [[TMP24:%.*]] = or disjoint i64 [[TMP10]], 1
; VF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP24]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP25]], align 8
-; VF4-NEXT: [[TMP33:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP33:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
; VF4-NEXT: [[TMP34:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP34]], align 8
; VF4-NEXT: [[TMP36:%.*]] = mul <4 x i64> [[WIDE_LOAD2]], [[TMP33]]
-; VF4-NEXT: [[TMP13:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP36]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP13]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP12]], <4 x i64> [[TMP36]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
@@ -666,16 +664,14 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <6 x i64>, ptr [[TMP4]], align 8
-; VF2-NEXT: [[TMP9:%.*]] = shufflevector <6 x i64> [[WIDE_VEC]], <6 x i64> poison, <2 x i32> <i32 0, i32 3>
-; VF2-NEXT: [[TMP18:%.*]] = shufflevector <6 x i64> [[WIDE_VEC]], <6 x i64> poison, <2 x i32> <i32 1, i32 4>
-; VF2-NEXT: [[TMP27:%.*]] = shufflevector <6 x i64> [[WIDE_VEC]], <6 x i64> poison, <2 x i32> <i32 2, i32 5>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP9:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 0
+; VF2-NEXT: [[TMP18:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 1
+; VF2-NEXT: [[TMP27:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC]], 2
; VF2-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP9]]
; VF2-NEXT: [[TMP19:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP18]]
; VF2-NEXT: [[TMP28:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP27]]
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP10]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP28]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i64> [[TMP11]], <6 x i64> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <6 x i64> @llvm.vector.interleave3.v6i64(<2 x i64> [[TMP10]], <2 x i64> [[TMP19]], <2 x i64> [[TMP28]])
; VF2-NEXT: store <6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -697,16 +693,14 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor) {
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
; VF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP6]], align 8
-; VF4-NEXT: [[TMP17:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; VF4-NEXT: [[TMP34:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; VF4-NEXT: [[TMP51:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64>, <4 x i64> } @llvm.vector.deinterleave3.v12i64(<12 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP17:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP34:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
+; VF4-NEXT: [[TMP51:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 2
; VF4-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP17]]
; VF4-NEXT: [[TMP35:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP34]]
; VF4-NEXT: [[TMP52:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP51]]
-; VF4-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP18]], <4 x i64> [[TMP35]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP52]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> [[TMP8]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP9]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i64> @llvm.vector.interleave3.v12i64(<4 x i64> [[TMP18]], <4 x i64> [[TMP35]], <4 x i64> [[TMP52]])
; VF4-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP6]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -755,12 +749,14 @@ define void @test_2xi32(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP7:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC]], 0
; VF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[TMP1]], i32 0
; VF2-NEXT: [[WIDE_VEC1:%.*]] = load <6 x i32>, ptr [[TMP8]], align 8
-; VF2-NEXT: [[TMP13:%.*]] = shufflevector <6 x i32> [[WIDE_VEC1]], <6 x i32> poison, <2 x i32> <i32 0, i32 3>
-; VF2-NEXT: [[TMP22:%.*]] = shufflevector <6 x i32> [[WIDE_VEC1]], <6 x i32> poison, <2 x i32> <i32 1, i32 4>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.vector.deinterleave3.v6i32(<6 x i32> [[WIDE_VEC1]])
+; VF2-NEXT: [[TMP13:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[TMP22:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP14:%.*]] = mul <2 x i32> [[TMP7]], [[TMP13]]
; VF2-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP14]], i64 0
; VF2-NEXT: store i32 [[TMP15]], ptr [[TMP8]], align 8
@@ -793,14 +789,16 @@ define void @test_2xi32(ptr noalias %data, ptr noalias %factor) {
; VF4-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 8
-; VF4-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP15:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 0
; VF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[TMP1]], i32 0
; VF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[TMP2]], i32 0
; VF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[TMP3]], i32 0
; VF4-NEXT: [[WIDE_VEC1:%.*]] = load <12 x i32>, ptr [[TMP16]], align 8
-; VF4-NEXT: [[TMP27:%.*]] = shufflevector <12 x i32> [[WIDE_VEC1]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; VF4-NEXT: [[TMP44:%.*]] = shufflevector <12 x i32> [[WIDE_VEC1]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC1]])
+; VF4-NEXT: [[TMP27:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[TMP44:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP28:%.*]] = mul <4 x i32> [[TMP15]], [[TMP27]]
; VF4-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0
; VF4-NEXT: store i32 [[TMP29]], ptr [[TMP16]], align 8
@@ -864,19 +862,18 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP3]], align 8
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> [[WIDE_VEC]])
+; VF2-NEXT: [[TMP7:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC]], 0
; VF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF2-NEXT: [[WIDE_VEC1:%.*]] = load <6 x i32>, ptr [[TMP9]], align 8
-; VF2-NEXT: [[TMP13:%.*]] = shufflevector <6 x i32> [[WIDE_VEC1]], <6 x i32> poison, <2 x i32> <i32 0, i32 3>
-; VF2-NEXT: [[TMP22:%.*]] = shufflevector <6 x i32> [[WIDE_VEC1]], <6 x i32> poison, <2 x i32> <i32 1, i32 4>
-; VF2-NEXT: [[TMP31:%.*]] = shufflevector <6 x i32> [[WIDE_VEC1]], <6 x i32> poison, <2 x i32> <i32 2, i32 5>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.vector.deinterleave3.v6i32(<6 x i32> [[WIDE_VEC1]])
+; VF2-NEXT: [[TMP13:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[TMP22:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 1
+; VF2-NEXT: [[TMP31:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 2
; VF2-NEXT: [[TMP14:%.*]] = mul <2 x i32> [[TMP7]], [[TMP13]]
; VF2-NEXT: [[TMP23:%.*]] = mul <2 x i32> [[TMP7]], [[TMP22]]
; VF2-NEXT: [[TMP32:%.*]] = mul <2 x i32> [[TMP7]], [[TMP31]]
-; VF2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP32]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; VF2-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP11]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i32> [[TMP10]], <6 x i32> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <6 x i32> @llvm.vector.interleave3.v6i32(<2 x i32> [[TMP14]], <2 x i32> [[TMP23]], <2 x i32> [[TMP32]])
; VF2-NEXT: store <6 x i32> [[INTERLEAVED_VEC]], ptr [[TMP9]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
@@ -895,19 +892,18 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP7]], align 8
-; VF4-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP15:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 0
; VF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[INDEX]], i32 0
; VF4-NEXT: [[WIDE_VEC1:%.*]] = load <12 x i32>, ptr [[TMP19]], align 8
-; VF4-NEXT: [[TMP27:%.*]] = shufflevector <12 x i32> [[WIDE_VEC1]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; VF4-NEXT: [[TMP44:%.*]] = shufflevector <12 x i32> [[WIDE_VEC1]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; VF4-NEXT: [[TMP61:%.*]] = shufflevector <12 x i32> [[WIDE_VEC1]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC1]])
+; VF4-NEXT: [[TMP27:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[TMP44:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
+; VF4-NEXT: [[TMP61:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 2
; VF4-NEXT: [[TMP28:%.*]] = mul <4 x i32> [[TMP15]], [[TMP27]]
; VF4-NEXT: [[TMP45:%.*]] = mul <4 x i32> [[TMP15]], [[TMP44]]
; VF4-NEXT: [[TMP62:%.*]] = mul <4 x i32> [[TMP15]], [[TMP61]]
-; VF4-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP28]], <4 x i32> [[TMP45]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP62]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; VF4-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i32> [[TMP10]], <12 x i32> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> [[TMP28]], <4 x i32> [[TMP45]], <4 x i32> [[TMP62]])
; VF4-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP19]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
@@ -987,8 +983,7 @@ define void @test_2xi64_sub_of_wide_loads(ptr noalias %data, ptr noalias %A, ptr
; VF4-NEXT: [[TMP5:%.*]] = sub <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; VF4-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP0]], 1
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP5]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; VF4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1040,8 +1035,7 @@ define void @test_2xi64_sub_of_wide_loads_ops_swapped(ptr noalias %data, ptr noa
; VF2-NEXT: [[TMP20:%.*]] = shl nsw i64 [[TMP0]], 1
; VF2-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP20]]
; VF2-NEXT: [[TMP15:%.*]] = sub <2 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT6]]
-; VF2-NEXT: [[TMP17:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC4:%.*]] = shufflevector <4 x i64> [[TMP17]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP13]], <2 x i64> [[TMP15]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC4]], ptr [[DATA_1]], align 8
; VF2-NEXT: [[IV_NEXT]] = add nuw i64 [[TMP0]], 2
; VF2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
@@ -1067,8 +1061,7 @@ define void @test_2xi64_sub_of_wide_loads_ops_swapped(ptr noalias %data, ptr noa
; VF4-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP0]], 1
; VF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
; VF4-NEXT: [[TMP8:%.*]] = sub <4 x i64> [[WIDE_LOAD1]], [[WIDE_LOAD]]
-; VF4-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP8]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; VF4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1122,8 +1115,7 @@ define void @test_2xi64_sub_of_wide_loads_with_different_base_ptrs(ptr noalias %
; VF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP8]], align 8
; VF2-NEXT: [[TMP10:%.*]] = sub <2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; VF2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP5]], <2 x i64> [[TMP10]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
; VF2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1151,8 +1143,7 @@ define void @test_2xi64_sub_of_wide_loads_with_different_base_ptrs(ptr noalias %
; VF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
; VF4-NEXT: [[TMP10:%.*]] = sub <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; VF4-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP10]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; VF4-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1225,16 +1216,17 @@ define void @multiple_store_groups_storing_same_wide_bin_op(ptr noalias %A, ptr
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP0]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP1:%.*]] = fadd contract <4 x double> [[STRIDED_VEC]], splat (double 2.000000e+01)
; VF4-NEXT: [[TMP2:%.*]] = fadd contract <4 x double> [[STRIDED_VEC1]], splat (double 2.000000e+01)
; VF4-NEXT: [[TMP3:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP4]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP1]], <4 x double> [[TMP2]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[TMP5:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP1]], <4 x double> [[TMP2]])
+; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC1]], ptr [[TMP5]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF4-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
@@ -1305,13 +1297,13 @@ define void @flag_mismatch_disjoint(ptr noalias %dst, ptr noalias %src) {
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = or disjoint <4 x i64> [[STRIDED_VEC]], splat (i64 1)
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; VF4-NEXT: [[TMP4:%.*]] = or <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
-; VF4-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1358,14 +1350,14 @@ define void @test_2xdouble_different_fcmp_predicates(ptr noalias %data) {
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[STRIDED_VEC]], zeroinitializer
; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = fcmp olt <2 x double> [[STRIDED_VEC1]], zeroinitializer
; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP4]], <2 x double> zeroinitializer, <2 x double> [[STRIDED_VEC1]]
-; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP3]], <2 x double> [[TMP5]])
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1386,14 +1378,14 @@ define void @test_2xdouble_different_fcmp_predicates(ptr noalias %data) {
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x double> [[STRIDED_VEC]], zeroinitializer
; VF4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC]]
; VF4-NEXT: [[TMP4:%.*]] = fcmp olt <4 x double> [[STRIDED_VEC1]], zeroinitializer
; VF4-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP3]], <4 x double> [[TMP5]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1440,14 +1432,14 @@ define void @test_2xi64_different_icmp_predicates(ptr noalias %data) {
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; VF2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 0
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC2]], 1
; VF2-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i64> [[STRIDED_VEC]], zeroinitializer
; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> zeroinitializer, <2 x i64> [[STRIDED_VEC]]
; VF2-NEXT: [[TMP4:%.*]] = icmp slt <2 x i64> [[STRIDED_VEC1]], zeroinitializer
; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP4]], <2 x i64> zeroinitializer, <2 x i64> [[STRIDED_VEC1]]
-; VF2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP5]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1468,14 +1460,14 @@ define void @test_2xi64_different_icmp_predicates(ptr noalias %data) {
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i64> [[STRIDED_VEC]], zeroinitializer
; VF4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> [[STRIDED_VEC]]
; VF4-NEXT: [[TMP4:%.*]] = icmp slt <4 x i64> [[STRIDED_VEC1]], zeroinitializer
; VF4-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> zeroinitializer, <4 x i64> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP3]], <4 x i64> [[TMP5]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1545,14 +1537,14 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x double> [[STRIDED_VEC]], zeroinitializer
; VF4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC]]
; VF4-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x double> [[STRIDED_VEC1]], zeroinitializer
; VF4-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x double> zeroinitializer, <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP3]], <4 x double> [[TMP5]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -1621,12 +1613,12 @@ define void @narrowed_fneg_intersects_fmf(ptr noalias %data) {
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 0
+; VF4-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC2]], 1
; VF4-NEXT: [[TMP2:%.*]] = fneg reassoc ninf <4 x double> [[STRIDED_VEC]]
; VF4-NEXT: [[TMP3:%.*]] = fneg ninf <4 x double> [[STRIDED_VEC1]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP4]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP2]], <4 x double> [[TMP3]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 5c9a3eac44fa0..c3deaa8890a8b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -37,10 +37,10 @@ define void @load_store_interleave_group(ptr noalias %data) {
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; VF4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[STRIDED_VEC]], <4 x i64> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP5:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP3]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -104,11 +104,11 @@ define void @load_store_interleave_group_different_objecs(ptr noalias %src, ptr
; VF4-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP1]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP6:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP1]]
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[STRIDED_VEC]], <4 x i64> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP4]], <4 x i64> [[TMP6]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -154,8 +154,7 @@ define void @single_wide_load_store_interleave_group(ptr noalias %src, ptr noali
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP7]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
; VF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP9]]
-; VF2-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC2:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC2:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD1]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC2]], ptr [[TMP8]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 2
; VF2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -177,8 +176,7 @@ define void @single_wide_load_store_interleave_group(ptr noalias %src, ptr noali
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP7]]
; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; VF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP9]]
-; VF4-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD1]], <4 x i64> [[WIDE_LOAD1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC2:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC2:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[WIDE_LOAD1]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC2]], ptr [[TMP8]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -239,7 +237,8 @@ define void @same_constant_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: store <8 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> zeroinitializer, <4 x i64> zeroinitializer)
+; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -278,7 +277,8 @@ define void @different_constants_store_interleave_group(i64 %x, i64 %y, ptr noal
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF2-NEXT: store <4 x i64> <i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> zeroinitializer, <2 x i64> splat (i64 1))
+; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -297,7 +297,8 @@ define void @different_constants_store_interleave_group(i64 %x, i64 %y, ptr noal
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: store <8 x i64> <i64 0, i64 1, i64 0, i64 1, i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> zeroinitializer, <4 x i64> splat (i64 1))
+; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF4-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -359,8 +360,7 @@ define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -404,8 +404,7 @@ define void @different_live_ins_store_interleave_group(i64 %x, i64 %y, ptr noali
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF2-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -429,8 +428,7 @@ define void @different_live_ins_store_interleave_group(i64 %x, i64 %y, ptr noali
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -495,8 +493,7 @@ define void @single_uniform_load_store_interleave_group(ptr noalias %src, ptr no
; VF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
; VF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; VF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
-; VF4-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -543,8 +540,7 @@ define void @multiple_uniform_load_store_interleave_group(ptr noalias %src.0, pt
; VF2-NEXT: [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
; VF2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
; VF2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
-; VF2-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]])
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -570,8 +566,7 @@ define void @multiple_uniform_load_store_interleave_group(ptr noalias %src.0, pt
; VF4-NEXT: [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
; VF4-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
; VF4-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; VF4-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]])
; VF4-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -636,14 +631,15 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP0]], align 8
-; VF4-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> [[WIDE_VEC]])
+; VF4-NEXT: [[TMP5:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 0
+; VF4-NEXT: [[TMP2:%.*]] = extractvalue { <4 x double>, <4 x double> } [[STRIDED_VEC]], 1
; VF4-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP5]], <4 x double> [[TMP2]])
; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP3:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
+; VF4-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <8 x double> @llvm.vector.interleave2.v8f64(<4 x double> [[TMP5]], <4 x double> [[TMP2]])
+; VF4-NEXT: store <8 x double> [[INTERLEAVED_VEC1]], ptr [[TMP3]], align 8
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF4-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
index f27ebb085dbec..d2ca9c73771fa 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
@@ -126,7 +126,8 @@ define void @test_stride2_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 2
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP4]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> splat (i32 5), [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP8]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index 2b9347b05e6bd..d3bda82bb3fa1 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -1502,8 +1502,9 @@ define i32 @reduction_interleave_group(i32 %n, ptr %arr) #0 {
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[TMP11]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 -4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[VEC_PHI]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])
@@ -1948,12 +1949,14 @@ define i64 @test_fir_q15(ptr %x, ptr %y, i32 %n) #0 {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP4]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC5]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[Y]], i32 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP6:%.*]] = sext <8 x i16> [[STRIDED_VEC3]] to <8 x i64>
; CHECK-NEXT: [[TMP7:%.*]] = sext <8 x i16> [[STRIDED_VEC]] to <8 x i64>
; CHECK-NEXT: [[TMP8:%.*]] = mul nsw <8 x i64> [[TMP6]], [[TMP7]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
index e5ed1f0d2b6a2..5ab801e8a04a7 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
@@ -65,7 +65,8 @@ define hidden void @pointer_phi_v4i32_add2(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -233,7 +234,8 @@ define hidden void @pointer_phi_v8i16_add2(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <8 x i16> [[TMP3]], ptr [[NEXT_GEP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -396,7 +398,8 @@ define hidden void @pointer_phi_v16i8_add2(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i32 [[OFFSET_IDX]]
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i8> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <16 x i8> [[TMP3]], ptr [[NEXT_GEP1]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
@@ -543,7 +546,8 @@ define hidden void @pointer_phi_v4f32_add2(ptr noalias nocapture readonly %A, pt
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -708,7 +712,8 @@ define hidden void @pointer_phi_v4half_add2(ptr noalias nocapture readonly %A, p
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x half>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x half> [[WIDE_VEC]], <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x half>, <8 x half> } @llvm.vector.deinterleave2.v16f16(<16 x half> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x half>, <8 x half> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <8 x half> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -769,7 +774,8 @@ define hidden void @pointer_phi_v4half_add3(ptr noalias nocapture readonly %A, p
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <24 x half>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <24 x half> [[WIDE_VEC]], <24 x half> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x half>, <8 x half>, <8 x half> } @llvm.vector.deinterleave3.v24f16(<24 x half> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x half>, <8 x half>, <8 x half> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <8 x half> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: store <8 x half> [[TMP2]], ptr [[NEXT_GEP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
index 7c9622aadb8f2..58a43e41322fe 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
@@ -48,61 +48,69 @@ define void @QLA_F3_r_veq_norm2_V(ptr noalias %r, ptr noalias %a, i32 %n) {
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [3 x { float, float }], ptr [[A]], i64 [[TMP6]], i64 0, i32 0
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [3 x { float, float }], ptr [[A]], i64 [[TMP7]], i64 0, i32 0
; CHECK-NEXT: [[WIDE_VEC35:%.*]] = load <12 x float>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[STRIDED_VEC36:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = shufflevector <12 x float> [[WIDE_VEC35]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC35]])
+; CHECK-NEXT: [[STRIDED_VEC36:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 3
+; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 4
+; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 5
; CHECK-NEXT: [[WIDE_VEC42:%.*]] = load <12 x float>, ptr [[TMP14]], align 8
-; CHECK-NEXT: [[STRIDED_VEC43:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC44:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC45:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC46:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC47:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC48:%.*]] = shufflevector <12 x float> [[WIDE_VEC42]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC42]])
+; CHECK-NEXT: [[STRIDED_VEC43:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 0
+; CHECK-NEXT: [[STRIDED_VEC44:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 1
+; CHECK-NEXT: [[STRIDED_VEC45:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 2
+; CHECK-NEXT: [[STRIDED_VEC46:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 3
+; CHECK-NEXT: [[STRIDED_VEC47:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 4
+; CHECK-NEXT: [[STRIDED_VEC48:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC10]], 5
; CHECK-NEXT: [[WIDE_VEC49:%.*]] = load <12 x float>, ptr [[TMP15]], align 8
-; CHECK-NEXT: [[STRIDED_VEC50:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC51:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC52:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC53:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC54:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC55:%.*]] = shufflevector <12 x float> [[WIDE_VEC49]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC12:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC49]])
+; CHECK-NEXT: [[STRIDED_VEC50:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 0
+; CHECK-NEXT: [[STRIDED_VEC51:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 1
+; CHECK-NEXT: [[STRIDED_VEC52:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 2
+; CHECK-NEXT: [[STRIDED_VEC53:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 3
+; CHECK-NEXT: [[STRIDED_VEC54:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 4
+; CHECK-NEXT: [[STRIDED_VEC55:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC12]], 5
; CHECK-NEXT: [[WIDE_VEC56:%.*]] = load <12 x float>, ptr [[TMP16]], align 8
-; CHECK-NEXT: [[STRIDED_VEC57:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC58:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC59:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC60:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC61:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC62:%.*]] = shufflevector <12 x float> [[WIDE_VEC56]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC14:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC56]])
+; CHECK-NEXT: [[STRIDED_VEC57:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 0
+; CHECK-NEXT: [[STRIDED_VEC58:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 1
+; CHECK-NEXT: [[STRIDED_VEC59:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 2
+; CHECK-NEXT: [[STRIDED_VEC60:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 3
+; CHECK-NEXT: [[STRIDED_VEC61:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 4
+; CHECK-NEXT: [[STRIDED_VEC62:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC14]], 5
; CHECK-NEXT: [[WIDE_VEC36:%.*]] = load <12 x float>, ptr [[TMP12]], align 8
-; CHECK-NEXT: [[STRIDED_VEC42:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC49:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC56:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC63:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC64:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC65:%.*]] = shufflevector <12 x float> [[WIDE_VEC36]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC16:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC36]])
+; CHECK-NEXT: [[STRIDED_VEC42:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 0
+; CHECK-NEXT: [[STRIDED_VEC49:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 1
+; CHECK-NEXT: [[STRIDED_VEC56:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 2
+; CHECK-NEXT: [[STRIDED_VEC63:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 3
+; CHECK-NEXT: [[STRIDED_VEC64:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 4
+; CHECK-NEXT: [[STRIDED_VEC65:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC16]], 5
; CHECK-NEXT: [[WIDE_VEC43:%.*]] = load <12 x float>, ptr [[TMP17]], align 8
-; CHECK-NEXT: [[STRIDED_VEC66:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC67:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC68:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC69:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC70:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC71:%.*]] = shufflevector <12 x float> [[WIDE_VEC43]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC18:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC43]])
+; CHECK-NEXT: [[STRIDED_VEC66:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 0
+; CHECK-NEXT: [[STRIDED_VEC67:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 1
+; CHECK-NEXT: [[STRIDED_VEC68:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 2
+; CHECK-NEXT: [[STRIDED_VEC69:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 3
+; CHECK-NEXT: [[STRIDED_VEC70:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 4
+; CHECK-NEXT: [[STRIDED_VEC71:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC18]], 5
; CHECK-NEXT: [[WIDE_VEC50:%.*]] = load <12 x float>, ptr [[TMP18]], align 8
-; CHECK-NEXT: [[STRIDED_VEC72:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC73:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC80:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC81:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC82:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC83:%.*]] = shufflevector <12 x float> [[WIDE_VEC50]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC20:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC50]])
+; CHECK-NEXT: [[STRIDED_VEC72:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 0
+; CHECK-NEXT: [[STRIDED_VEC73:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 1
+; CHECK-NEXT: [[STRIDED_VEC80:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 2
+; CHECK-NEXT: [[STRIDED_VEC81:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 3
+; CHECK-NEXT: [[STRIDED_VEC82:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 4
+; CHECK-NEXT: [[STRIDED_VEC83:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC20]], 5
; CHECK-NEXT: [[WIDE_VEC57:%.*]] = load <12 x float>, ptr [[TMP19]], align 8
-; CHECK-NEXT: [[STRIDED_VEC84:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC85:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC86:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC87:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC88:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC89:%.*]] = shufflevector <12 x float> [[WIDE_VEC57]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC57]])
+; CHECK-NEXT: [[STRIDED_VEC84:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 0
+; CHECK-NEXT: [[STRIDED_VEC85:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 1
+; CHECK-NEXT: [[STRIDED_VEC86:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 2
+; CHECK-NEXT: [[STRIDED_VEC87:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 3
+; CHECK-NEXT: [[STRIDED_VEC88:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 4
+; CHECK-NEXT: [[STRIDED_VEC89:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC22]], 5
; CHECK-NEXT: [[TMP64:%.*]] = fmul fast <2 x float> [[STRIDED_VEC36]], [[STRIDED_VEC36]]
; CHECK-NEXT: [[TMP97:%.*]] = fmul fast <2 x float> [[STRIDED_VEC43]], [[STRIDED_VEC43]]
; CHECK-NEXT: [[TMP98:%.*]] = fmul fast <2 x float> [[STRIDED_VEC50]], [[STRIDED_VEC50]]
@@ -252,12 +260,13 @@ define void @QLA_F3_r_veq_norm2_V(ptr noalias %r, ptr noalias %a, i32 %n) {
; CHECK-NEXT: [[VEC_PHI72:%.*]] = phi <2 x double> [ [[TMP138]], %[[VEC_EPILOG_PH]] ], [ [[TMP176:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX5_REALP:%.*]] = getelementptr inbounds [3 x { float, float }], ptr [[A]], i64 [[INDVARS_IV]], i64 0, i32 0
; CHECK-NEXT: [[WIDE_VEC73:%.*]] = load <12 x float>, ptr [[ARRAYIDX5_REALP]], align 8
-; CHECK-NEXT: [[STRIDED_VEC74:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 0, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC75:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 1, i32 7>
-; CHECK-NEXT: [[STRIDED_VEC76:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 2, i32 8>
-; CHECK-NEXT: [[STRIDED_VEC77:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 3, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC78:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 4, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC79:%.*]] = shufflevector <12 x float> [[WIDE_VEC73]], <12 x float> poison, <2 x i32> <i32 5, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC33:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave6.v12f32(<12 x float> [[WIDE_VEC73]])
+; CHECK-NEXT: [[STRIDED_VEC74:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 0
+; CHECK-NEXT: [[STRIDED_VEC75:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 1
+; CHECK-NEXT: [[STRIDED_VEC76:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 2
+; CHECK-NEXT: [[STRIDED_VEC77:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 3
+; CHECK-NEXT: [[STRIDED_VEC78:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 4
+; CHECK-NEXT: [[STRIDED_VEC79:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC33]], 5
; CHECK-NEXT: [[TMP140:%.*]] = fmul fast <2 x float> [[STRIDED_VEC74]], [[STRIDED_VEC74]]
; CHECK-NEXT: [[TMP141:%.*]] = fmul fast <2 x float> [[STRIDED_VEC75]], [[STRIDED_VEC75]]
; CHECK-NEXT: [[TMP177:%.*]] = fadd fast <2 x float> [[TMP141]], [[TMP140]]
@@ -282,11 +291,11 @@ define void @QLA_F3_r_veq_norm2_V(ptr noalias %r, ptr noalias %a, i32 %n) {
; CHECK-NEXT: br i1 [[CMP_N81]], label %[[FOR_COND_FOR_END13_CRIT_EDGE]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC70]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX81:%.*]] = phi double [ [[TMP186]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP158]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX36:%.*]] = phi double [ [[TMP186]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP158]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
; CHECK: [[FOR_COND1_PREHEADER]]:
; CHECK-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_COND1_PREHEADER]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT: [[SUM_026:%.*]] = phi double [ [[ADD10_2:%.*]], %[[FOR_COND1_PREHEADER]] ], [ [[BC_MERGE_RDX81]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[SUM_026:%.*]] = phi double [ [[ADD10_2:%.*]], %[[FOR_COND1_PREHEADER]] ], [ [[BC_MERGE_RDX36]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX5_REALP1:%.*]] = getelementptr inbounds [3 x { float, float }], ptr [[A]], i64 [[INDVARS_IV1]], i64 0, i32 0
; CHECK-NEXT: [[ARRAYIDX5_REAL:%.*]] = load float, ptr [[ARRAYIDX5_REALP1]], align 8
; CHECK-NEXT: [[ARRAYIDX5_IMAGP:%.*]] = getelementptr inbounds [3 x { float, float }], ptr [[A]], i64 [[INDVARS_IV1]], i64 0, i32 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index cc65d7903c5a7..30599e4d10a11 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -45,12 +45,12 @@ define void @load_store_factor2_i32(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 1
; FIXED-NEXT: [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
-; FIXED-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP3]])
; FIXED-NEXT: store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; FIXED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -157,12 +157,12 @@ define void @load_store_factor2_i64(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP3]])
; FIXED-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -271,16 +271,14 @@ define void @load_store_factor3_i32(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP1]], align 4
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = call { <8 x i32>, <8 x i32>, <8 x i32> } @llvm.vector.deinterleave3.v24i32(<24 x i32> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC3]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC3]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC3]], 2
; FIXED-NEXT: [[TMP2:%.*]] = add <8 x i32> [[STRIDED_VEC]], splat (i32 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <8 x i32> [[STRIDED_VEC1]], splat (i32 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC2]], splat (i32 3)
-; FIXED-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP7]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> [[TMP4]])
; FIXED-NEXT: store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -397,16 +395,14 @@ define void @load_store_factor3_i64(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i64>, <4 x i64>, <4 x i64> } @llvm.vector.deinterleave3.v12i64(<12 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 2
; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP7:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP7]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i64> @llvm.vector.interleave3.v12i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <4 x i64> [[TMP4]])
; FIXED-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -654,22 +650,18 @@ define void @load_store_factor5(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 5
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <10 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 0, i32 5>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 1, i32 6>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 2, i32 7>
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 3, i32 8>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <10 x i64> [[WIDE_VEC]], <10 x i64> poison, <2 x i32> <i32 4, i32 9>
+; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave5.v10i64(<10 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 2
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 3
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 4
; FIXED-NEXT: [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
; FIXED-NEXT: [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
; FIXED-NEXT: [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
-; FIXED-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <10 x i64> [[TMP11]], <10 x i64> poison, <10 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 1, i32 3, i32 5, i32 7, i32 9>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <10 x i64> @llvm.vector.interleave5.v10i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]])
; FIXED-NEXT: store <10 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -808,25 +800,20 @@ define void @load_store_factor6(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 6
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 0, i32 6>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 1, i32 7>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 2, i32 8>
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 3, i32 9>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 4, i32 10>
-; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <2 x i32> <i32 5, i32 11>
+; FIXED-NEXT: [[STRIDED_VEC6:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave6.v12i64(<12 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 2
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 3
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 4
+; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC6]], 5
; FIXED-NEXT: [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
; FIXED-NEXT: [[TMP5:%.*]] = add <2 x i64> [[STRIDED_VEC3]], splat (i64 4)
; FIXED-NEXT: [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
; FIXED-NEXT: [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
-; FIXED-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i64> @llvm.vector.interleave6.v12i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <2 x i64> [[TMP7]])
; FIXED-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -975,13 +962,14 @@ define void @load_store_factor7(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 7
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <14 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 0, i32 7>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 1, i32 8>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 2, i32 9>
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 3, i32 10>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 4, i32 11>
-; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 5, i32 12>
-; FIXED-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <14 x i64> [[WIDE_VEC]], <14 x i64> poison, <2 x i32> <i32 6, i32 13>
+; FIXED-NEXT: [[STRIDED_VEC7:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave7.v14i64(<14 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 2
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 3
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 4
+; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 5
+; FIXED-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC7]], 6
; FIXED-NEXT: [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
@@ -989,15 +977,7 @@ define void @load_store_factor7(ptr %p) {
; FIXED-NEXT: [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC4]], splat (i64 5)
; FIXED-NEXT: [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
; FIXED-NEXT: [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
-; FIXED-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP11]], <4 x i64> [[TMP13]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; FIXED-NEXT: [[TMP15:%.*]] = shufflevector <6 x i64> [[TMP14]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
-; FIXED-NEXT: [[TMP16:%.*]] = shufflevector <8 x i64> [[TMP12]], <8 x i64> [[TMP15]], <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <14 x i64> [[TMP16]], <14 x i64> poison, <14 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <14 x i64> @llvm.vector.interleave7.v14i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <2 x i64> [[TMP8]])
; FIXED-NEXT: store <14 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -1156,14 +1136,15 @@ define void @load_store_factor8(ptr %p) {
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 3
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
-; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
-; FIXED-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
-; FIXED-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
+; FIXED-NEXT: [[STRIDED_VEC8:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave8.v16i64(<16 x i64> [[WIDE_VEC]])
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 0
+; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 1
+; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 2
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 3
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 4
+; FIXED-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 5
+; FIXED-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 6
+; FIXED-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[STRIDED_VEC8]], 7
; FIXED-NEXT: [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
; FIXED-NEXT: [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 2)
; FIXED-NEXT: [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC2]], splat (i64 3)
@@ -1172,14 +1153,7 @@ define void @load_store_factor8(ptr %p) {
; FIXED-NEXT: [[TMP7:%.*]] = add <2 x i64> [[STRIDED_VEC5]], splat (i64 6)
; FIXED-NEXT: [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC6]], splat (i64 7)
; FIXED-NEXT: [[TMP9:%.*]] = add <2 x i64> [[STRIDED_VEC7]], splat (i64 8)
-; FIXED-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP15:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP16:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP16]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i64> @llvm.vector.interleave8.v16i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <2 x i64> [[TMP8]], <2 x i64> [[TMP9]])
; FIXED-NEXT: store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -1332,8 +1306,9 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
; FIXED-NEXT: [[TMP2:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP2]]
; FIXED-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC2]])
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC]], 0
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC]], 1
; FIXED-NEXT: [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
; FIXED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
; FIXED-NEXT: store <8 x i32> [[TMP6]], ptr [[TMP7]], align 4
@@ -1439,8 +1414,9 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
; FIXED-NEXT: [[TMP2:%.*]] = shl i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP2]]
; FIXED-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; FIXED-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC2]])
+; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 0
+; FIXED-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC]], 1
; FIXED-NEXT: [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
; FIXED-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
; FIXED-NEXT: store <4 x i64> [[TMP6]], ptr [[TMP7]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll
index 2ab60341399ef..abbeac144c96d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll
@@ -18,8 +18,7 @@ define void @store_factor_2_with_tail_gap(i64 %n, ptr %a) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i64> [[VEC_IND]], <16 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i64> [[TMP2]], <32 x i64> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <32 x i64> @llvm.vector.interleave2.v32i64(<16 x i64> [[VEC_IND]], <16 x i64> poison)
; CHECK-NEXT: call void @llvm.masked.store.v32i64.p0(<32 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP1]], <32 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 16)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index b50a95f466aae..4e3304dd77827 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -501,10 +501,7 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; NO-VP-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i32 [[INDEX]], i32 0
-; NO-VP-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC_IND]], <8 x i32> [[VEC_IND]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; NO-VP-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC_IND]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; NO-VP-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; NO-VP-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[TMP3]], <32 x i32> poison, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; NO-VP-NEXT: [[INTERLEAVED_VEC:%.*]] = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> [[VEC_IND]], <8 x i32> [[VEC_IND]], <8 x i32> [[VEC_IND]], <8 x i32> poison)
; NO-VP-NEXT: call void @llvm.masked.store.v32i32.p0(<32 x i32> [[INTERLEAVED_VEC]], ptr align 4 [[TMP0]], <32 x i1> <i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false>)
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll
index 7f0ba19d3e7a8..79dea7a8414a6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll
@@ -137,12 +137,13 @@ define void @replicate_sext(i32 %N, ptr %dst, ptr %src) #0 {
; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[OFFSET_IDX]] to i64
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr nusw i32, ptr [[SRC]], i64 [[TMP22]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP23]], align 4, !alias.scope [[META4:![0-9]+]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP24:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP28:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[TMP25:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[DST]], <4 x i64> [[TMP25]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC]], <4 x ptr> align 4 [[TMP27]], <4 x i1> splat (i1 true)), !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC9]], <4 x ptr> align 4 [[TMP27]], <4 x i1> splat (i1 true)), !alias.scope [[META7]], !noalias [[META4]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP24]], <4 x ptr> align 4 [[TMP27]], <4 x i1> splat (i1 true)), !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP28]], <4 x ptr> align 4 [[TMP27]], <4 x i1> splat (i1 true)), !alias.scope [[META7]], !noalias [[META4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 12)
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -151,13 +152,13 @@ define void @replicate_sext(i32 %N, ptr %dst, ptr %src) #0 {
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[IV_1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_2_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_2_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[IV_2_EXT:%.*]] = sext i32 [[IV_2]] to i64
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr nusw i32, ptr [[SRC]], i64 [[IV_2_EXT]]
; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index 08ba28d85ccd7..f6d99d16b26a1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -39,12 +39,14 @@ define void @PR31671(float %x, ptr %d) #0 {
; CHECK-NEXT: [[TMP6:%.*]] = mul nuw i64 [[INDEX]], 5
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[DATA:%.*]], ptr [[D]], i64 0, i32 3, i64 [[TMP6]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <80 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <80 x float> [[WIDE_VEC]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <16 x float>, <16 x float>, <16 x float>, <16 x float>, <16 x float> } @llvm.vector.deinterleave5.v80f32(<80 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <16 x float>, <16 x float>, <16 x float>, <16 x float>, <16 x float> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP1:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, <16 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x ptr> [[TMP2]], i64 0
; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <80 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <80 x float> [[WIDE_VEC1]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <16 x float>, <16 x float>, <16 x float>, <16 x float>, <16 x float> } @llvm.vector.deinterleave5.v80f32(<80 x float> [[WIDE_VEC1]])
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <16 x float>, <16 x float>, <16 x float>, <16 x float>, <16 x float> } [[STRIDED_VEC3]], 0
; CHECK-NEXT: [[TMP4:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP1]]
; CHECK-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP4]], <16 x ptr> align 4 [[TMP2]], <16 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -78,13 +80,17 @@ define void @PR31671(float %x, ptr %d) #0 {
; FORCE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP4]]
; FORCE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP6]]
; FORCE-NEXT: [[WIDE_VEC:%.*]] = load <10 x float>, ptr [[TMP8]], align 4
-; FORCE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <10 x float> [[WIDE_VEC]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC]])
+; FORCE-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC1]], 0
; FORCE-NEXT: [[WIDE_VEC1:%.*]] = load <10 x float>, ptr [[TMP9]], align 4
-; FORCE-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <10 x float> [[WIDE_VEC1]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC3:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC1]])
+; FORCE-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC3]], 0
; FORCE-NEXT: [[WIDE_VEC3:%.*]] = load <10 x float>, ptr [[TMP10]], align 4
-; FORCE-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <10 x float> [[WIDE_VEC3]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC5:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC3]])
+; FORCE-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC5]], 0
; FORCE-NEXT: [[WIDE_VEC5:%.*]] = load <10 x float>, ptr [[TMP11]], align 4
-; FORCE-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <10 x float> [[WIDE_VEC5]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC7:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC5]])
+; FORCE-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC7]], 0
; FORCE-NEXT: [[TMP12:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
; FORCE-NEXT: [[TMP13:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC2]]
; FORCE-NEXT: [[TMP14:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
@@ -98,13 +104,17 @@ define void @PR31671(float %x, ptr %d) #0 {
; FORCE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP6]]
; FORCE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP7]]
; FORCE-NEXT: [[WIDE_VEC7:%.*]] = load <10 x float>, ptr [[TMP16]], align 4
-; FORCE-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <10 x float> [[WIDE_VEC7]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC9:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC7]])
+; FORCE-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC9]], 0
; FORCE-NEXT: [[WIDE_VEC9:%.*]] = load <10 x float>, ptr [[TMP18]], align 4
-; FORCE-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <10 x float> [[WIDE_VEC9]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC11:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC9]])
+; FORCE-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC11]], 0
; FORCE-NEXT: [[WIDE_VEC11:%.*]] = load <10 x float>, ptr [[TMP20]], align 4
-; FORCE-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <10 x float> [[WIDE_VEC11]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC13:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC11]])
+; FORCE-NEXT: [[STRIDED_VEC12:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 0
; FORCE-NEXT: [[WIDE_VEC13:%.*]] = load <10 x float>, ptr [[TMP22]], align 4
-; FORCE-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <10 x float> [[WIDE_VEC13]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT: [[STRIDED_VEC15:%.*]] = call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave5.v10f32(<10 x float> [[WIDE_VEC13]])
+; FORCE-NEXT: [[STRIDED_VEC14:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC15]], 0
; FORCE-NEXT: [[TMP24:%.*]] = fadd <2 x float> [[STRIDED_VEC8]], [[TMP12]]
; FORCE-NEXT: [[TMP25:%.*]] = fadd <2 x float> [[STRIDED_VEC10]], [[TMP13]]
; FORCE-NEXT: [[TMP26:%.*]] = fadd <2 x float> [[STRIDED_VEC12]], [[TMP14]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
index f0c94e0a78629..70d27aa4f0ef0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
@@ -411,13 +411,17 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP10]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x double>, ptr [[TMP11]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x double>, <4 x double>, <4 x double>, <4 x double> } @llvm.vector.deinterleave4.v16f64(<16 x double> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x double>, <4 x double>, <4 x double>, <4 x double> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <16 x double>, ptr [[TMP12]], align 8
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x double> [[WIDE_VEC2]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = call { <4 x double>, <4 x double>, <4 x double>, <4 x double> } @llvm.vector.deinterleave4.v16f64(<16 x double> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x double>, <4 x double>, <4 x double>, <4 x double> } [[STRIDED_VEC4]], 0
; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <16 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <16 x double> [[WIDE_VEC4]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = call { <4 x double>, <4 x double>, <4 x double>, <4 x double> } @llvm.vector.deinterleave4.v16f64(<16 x double> [[WIDE_VEC4]])
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x double>, <4 x double>, <4 x double>, <4 x double> } [[STRIDED_VEC6]], 0
; CHECK-NEXT: [[WIDE_VEC6:%.*]] = load <16 x double>, ptr [[TMP14]], align 8
-; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x double> [[WIDE_VEC6]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[STRIDED_VEC8:%.*]] = call { <4 x double>, <4 x double>, <4 x double>, <4 x double> } @llvm.vector.deinterleave4.v16f64(<16 x double> [[WIDE_VEC6]])
+; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x double>, <4 x double>, <4 x double>, <4 x double> } [[STRIDED_VEC8]], 0
; CHECK-NEXT: [[TMP15:%.*]] = fcmp oeq <4 x double> [[STRIDED_VEC]], zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = fcmp oeq <4 x double> [[STRIDED_VEC3]], zeroinitializer
; CHECK-NEXT: [[TMP17:%.*]] = fcmp oeq <4 x double> [[STRIDED_VEC5]], zeroinitializer
@@ -574,7 +578,8 @@ define void @cost_duplicate_recipe_for_sinking(ptr %A, i64 %N) #2 {
; CHECK-NEXT: [[TMP75:%.*]] = shl nsw i64 [[INDEX39]], 2
; CHECK-NEXT: [[TMP76:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP75]]
; CHECK-NEXT: [[WIDE_VEC40:%.*]] = load <16 x double>, ptr [[TMP76]], align 8
-; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = shufflevector <16 x double> [[WIDE_VEC40]], <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[STRIDED_VEC42:%.*]] = call { <4 x double>, <4 x double>, <4 x double>, <4 x double> } @llvm.vector.deinterleave4.v16f64(<16 x double> [[WIDE_VEC40]])
+; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = extractvalue { <4 x double>, <4 x double>, <4 x double>, <4 x double> } [[STRIDED_VEC42]], 0
; CHECK-NEXT: [[TMP77:%.*]] = fcmp oeq <4 x double> [[STRIDED_VEC41]], zeroinitializer
; CHECK-NEXT: [[TMP78:%.*]] = extractelement <4 x i1> [[TMP77]], i64 0
; CHECK-NEXT: br i1 [[TMP78]], label [[PRED_STORE_IF42:%.*]], label [[PRED_STORE_CONTINUE43:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index cf081d726faaa..2644bf67e0f33 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -379,7 +379,8 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x ptr> [[TMP27]], ptr [[TMP24]], i64 3
; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x ptr> [[TMP20]], <4 x ptr> [[TMP28]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP21]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x float> [[STRIDED_VEC]], i64 3
; CHECK-NEXT: store float [[TMP30]], ptr [[C:%.*]], align 4
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x ptr> [[TMP29]], i64 3
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index aa5a0f5be24be..f9f016ff0f8de 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -21,17 +21,13 @@ define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noali
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[P_INVAR]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLAT]], <2 x float> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x float> @llvm.vector.interleave4.v8f32(<2 x float> [[BROADCAST_SPLAT]], <2 x float> [[BROADCAST_SPLAT2]], <2 x float> zeroinitializer, <2 x float> zeroinitializer)
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[P_INVAR]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[DST_2]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLAT4]], <2 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = call <8 x float> @llvm.vector.interleave4.v8f32(<2 x float> [[BROADCAST_SPLAT4]], <2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC5]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -269,16 +265,17 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[TMP52:%.*]] = lshr exact i64 [[TMP51]], 1
; CHECK-NEXT: [[TMP53:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[TMP52]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP53]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC41:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP64:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP65:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP64]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP54:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[WIDE_GEP42:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP54]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP42]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP55:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[WIDE_GEP43:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP55]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[STRIDED_VEC41]], <4 x ptr> align 4 [[WIDE_GEP43]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP65]], <4 x ptr> align 4 [[WIDE_GEP43]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
; CHECK-NEXT: [[TMP56:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: [[WIDE_GEP44:%.*]] = getelementptr i32, ptr [[A]], <4 x i64> [[TMP56]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP44]], <4 x i1> splat (i1 true)), !alias.scope [[META6]]
@@ -397,7 +394,8 @@ define void @interleave_store_double_i64(ptr %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 4.940660e-324>, ptr [[DST]], align 8
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> zeroinitializer, <2 x double> <double 0.000000e+00, double 4.940660e-324>)
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -498,7 +496,8 @@ define void @interleave_store_i64_double_2(ptr %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 4.940660e-324, double 0.000000e+00>, ptr [[DST]], align 8
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> <double 0.000000e+00, double 4.940660e-324>, <2 x double> zeroinitializer)
+; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
index 30c46504f2495..17782ca8f68f6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
@@ -28,8 +28,7 @@ define void @test_pr55375_interleave_opaque_ptr(ptr %start, ptr %end) {
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[END]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <2 x i64> <i64 0, i64 16>
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x ptr> [[TMP10]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x ptr> splat (ptr null), <2 x ptr> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[TMP12]], <4 x ptr> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> splat (ptr null), <2 x ptr> [[TMP10]])
; CHECK-NEXT: store <4 x ptr> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
index 445b59c388742..89945c3ded9a1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
@@ -56,17 +56,21 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP19]], i32 -4
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP20]], i32 -4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP21]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC16:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC16:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[WIDE_VEC17:%.*]] = load <8 x i32>, ptr [[TMP22]], align 4
-; CHECK-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i32> [[WIDE_VEC17]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC17]])
+; CHECK-NEXT: [[STRIDED_VEC18:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 0
+; CHECK-NEXT: [[STRIDED_VEC19:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC17]], 1
; CHECK-NEXT: [[WIDE_VEC20:%.*]] = load <8 x i32>, ptr [[TMP23]], align 4
-; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = shufflevector <8 x i32> [[WIDE_VEC20]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = shufflevector <8 x i32> [[WIDE_VEC20]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC20:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC20]])
+; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC20]], 0
+; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC20]], 1
; CHECK-NEXT: [[WIDE_VEC23:%.*]] = load <8 x i32>, ptr [[TMP24]], align 4
-; CHECK-NEXT: [[STRIDED_VEC24:%.*]] = shufflevector <8 x i32> [[WIDE_VEC23]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = shufflevector <8 x i32> [[WIDE_VEC23]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC23]])
+; CHECK-NEXT: [[STRIDED_VEC24:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC23]], 0
+; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC23]], 1
; CHECK-NEXT: [[TMP25:%.*]] = add <4 x i32> [[STRIDED_VEC16]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP30:%.*]] = add <4 x i32> [[STRIDED_VEC19]], [[STRIDED_VEC18]]
; CHECK-NEXT: [[TMP35:%.*]] = add <4 x i32> [[STRIDED_VEC22]], [[STRIDED_VEC21]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-sink-store-across-load.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-sink-store-across-load.ll
index fca057e33337a..1f11c89c444b3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-sink-store-across-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-sink-store-across-load.ll
@@ -22,7 +22,8 @@ define void @avoid_sinking_store_across_load(ptr %arr) {
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <4 x i64> [[VEC_IND2]]
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <4 x i64> [[TMP5]]
@@ -30,8 +31,9 @@ define void @avoid_sinking_store_across_load(ptr %arr) {
; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[STRIDED_VEC]], splat (i32 25)
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP7]], <4 x ptr> align 4 [[TMP6]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <12 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <12 x i32> [[WIDE_VEC4]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <12 x i32> [[WIDE_VEC4]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC4]])
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 0
+; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 1
; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC5]]
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP9]], <4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
index ffddbbfcc81ca..25f6f2af2c286 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
@@ -19,11 +19,13 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
; SSE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP1]]
; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP9]]
; SSE-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
-; SSE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; SSE-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 0
+; SSE-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 1
; SSE-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; SSE-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC2]])
+; SSE-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; SSE-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; SSE-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; SSE-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
@@ -57,17 +59,21 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
; AVX1-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
; AVX1-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP19]]
; AVX1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4
-; AVX1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; AVX1-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 0
+; AVX1-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 1
; AVX1-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
-; AVX1-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC2]])
+; AVX1-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; AVX1-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; AVX1-NEXT: [[WIDE_VEC5:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
-; AVX1-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <8 x i32> [[WIDE_VEC5]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x i32> [[WIDE_VEC5]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC8:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC5]])
+; AVX1-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC8]], 0
+; AVX1-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC8]], 1
; AVX1-NEXT: [[WIDE_VEC8:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
-; AVX1-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC11:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC8]])
+; AVX1-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC11]], 0
+; AVX1-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC11]], 1
; AVX1-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; AVX1-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
; AVX1-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC6]]
@@ -107,17 +113,21 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP5]]
; AVX2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP19]]
; AVX2-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP6]], align 4
-; AVX2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC5:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC]])
+; AVX2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC5]], 0
+; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC5]], 1
; AVX2-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP7]], align 4
-; AVX2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC2]])
+; AVX2-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 0
+; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 1
; AVX2-NEXT: [[WIDE_VEC5:%.*]] = load <16 x i32>, ptr [[TMP8]], align 4
-; AVX2-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC8:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC5]])
+; AVX2-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC8]], 0
+; AVX2-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC8]], 1
; AVX2-NEXT: [[WIDE_VEC8:%.*]] = load <16 x i32>, ptr [[TMP9]], align 4
-; AVX2-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <16 x i32> [[WIDE_VEC8]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <16 x i32> [[WIDE_VEC8]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC11:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC8]])
+; AVX2-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC11]], 0
+; AVX2-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC11]], 1
; AVX2-NEXT: [[TMP10:%.*]] = add nsw <8 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; AVX2-NEXT: [[TMP11:%.*]] = add nsw <8 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
; AVX2-NEXT: [[TMP12:%.*]] = add nsw <8 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC6]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll b/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
index b6fd369f76207..03dd5bb7e87ab 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
@@ -63,7 +63,7 @@ define void @test_tc_18(ptr noalias %src, ptr noalias %dst) {
; CHECK: middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -75,7 +75,7 @@ define void @test_tc_18(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store <2 x i8> [[WIDE_LOAD2]], ptr [[TMP9]], align 64
; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 18
-; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
@@ -89,7 +89,7 @@ define void @test_tc_18(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64
; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 18
-; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -125,7 +125,7 @@ define void @test_tc_19(ptr noalias %src, ptr noalias %dst) {
; CHECK: middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF4]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -137,7 +137,7 @@ define void @test_tc_19(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store <2 x i8> [[WIDE_LOAD2]], ptr [[TMP9]], align 64
; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 18
-; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
@@ -151,7 +151,7 @@ define void @test_tc_19(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64
; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 19
-; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -199,7 +199,7 @@ define void @test_tc_20(ptr noalias %src, ptr noalias %dst) {
; CHECK: middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -211,7 +211,7 @@ define void @test_tc_20(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD5]], ptr [[TMP15]], align 64
; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT6]], 20
-; CHECK-NEXT: br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
@@ -225,7 +225,7 @@ define void @test_tc_20(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64
; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 20
-; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -256,12 +256,13 @@ define void @limit_main_loop_vf_to_avoid_dead_main_vector_loop(ptr noalias %src,
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [3 x i8], ptr [[SRC:%.*]], i64 [[INDEX]], i64 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <24 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <24 x i8> [[WIDE_VEC]], <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.vector.deinterleave3.v24i8(<24 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <8 x i8> [[STRIDED_VEC]], ptr [[TMP3]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH:%.*]]
; CHECK: scalar.ph:
@@ -274,7 +275,7 @@ define void @limit_main_loop_vf_to_avoid_dead_main_vector_loop(ptr noalias %src,
; CHECK-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 32
-; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index f1b49668f2b10..1a49e0cf4c122 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -191,7 +191,8 @@ define void @test_scalar_cost_single_store_loop_varying_cond(ptr %dst, ptr noali
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i32> [[WIDE_VEC]], <32 x i32> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i32>, <8 x i32>, <8 x i32>, <8 x i32> } @llvm.vector.deinterleave4.v32i32(<32 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <8 x i32> [[STRIDED_VEC]], splat (i32 123)
; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
index e546743472b71..c287a92b14e83 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr47437.ll
@@ -23,13 +23,15 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; SSE2-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; SSE2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[S1:%.*]], i64 [[TMP1]]
; SSE2-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; SSE2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC]])
+; SSE2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC5]], 0
+; SSE2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC5]], 1
; SSE2-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32>
; SSE2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[S2:%.*]], i64 [[TMP1]]
; SSE2-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2
-; SSE2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i16> [[WIDE_VEC2]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i16> [[WIDE_VEC2]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC2]])
+; SSE2-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC2]], 0
+; SSE2-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC2]], 1
; SSE2-NEXT: [[TMP7:%.*]] = sext <4 x i16> [[STRIDED_VEC3]] to <4 x i32>
; SSE2-NEXT: [[TMP8:%.*]] = mul nsw <4 x i32> [[TMP7]], [[TMP4]]
; SSE2-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[STRIDED_VEC1]] to <4 x i32>
@@ -96,21 +98,25 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; SSE41-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[S1:%.*]], i64 [[TMP2]]
; SSE41-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP3]]
; SSE41-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2
-; SSE41-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE41-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE41-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC]])
+; SSE41-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC1]], 0
+; SSE41-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC1]], 1
; SSE41-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2
-; SSE41-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i16> [[WIDE_VEC1]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE41-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i16> [[WIDE_VEC1]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE41-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC1]])
+; SSE41-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC5]], 0
+; SSE41-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC5]], 1
; SSE41-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32>
; SSE41-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[STRIDED_VEC2]] to <4 x i32>
; SSE41-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[S2:%.*]], i64 [[TMP2]]
; SSE41-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[S2]], i64 [[TMP3]]
; SSE41-NEXT: [[WIDE_VEC5:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2
-; SSE41-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x i16> [[WIDE_VEC5]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE41-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x i16> [[WIDE_VEC5]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE41-NEXT: [[STRIDED_VEC11:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC5]])
+; SSE41-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC11]], 0
+; SSE41-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC11]], 1
; SSE41-NEXT: [[WIDE_VEC6:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2
-; SSE41-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <8 x i16> [[WIDE_VEC6]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; SSE41-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x i16> [[WIDE_VEC6]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE41-NEXT: [[STRIDED_VEC6:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC6]])
+; SSE41-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC6]], 0
+; SSE41-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC6]], 1
; SSE41-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[STRIDED_VEC7]] to <4 x i32>
; SSE41-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[STRIDED_VEC8]] to <4 x i32>
; SSE41-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP14]], [[TMP8]]
@@ -194,17 +200,21 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX1-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP5]]
; AVX1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP6]]
; AVX1-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2
-; AVX1-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <8 x i16> [[WIDE_VEC2]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <8 x i16> [[WIDE_VEC2]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC2]])
+; AVX1-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC]], 0
+; AVX1-NEXT: [[STRIDED_VEC9:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC]], 1
; AVX1-NEXT: [[WIDE_VEC3:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2
-; AVX1-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <8 x i16> [[WIDE_VEC3]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <8 x i16> [[WIDE_VEC3]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC3]])
+; AVX1-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC3]], 0
+; AVX1-NEXT: [[STRIDED_VEC10:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC3]], 1
; AVX1-NEXT: [[WIDE_VEC6:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2
-; AVX1-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x i16> [[WIDE_VEC6]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <8 x i16> [[WIDE_VEC6]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC14:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC6]])
+; AVX1-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC14]], 0
+; AVX1-NEXT: [[STRIDED_VEC8:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC14]], 1
; AVX1-NEXT: [[WIDE_VEC9:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2
-; AVX1-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <8 x i16> [[WIDE_VEC9]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <8 x i16> [[WIDE_VEC9]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC16:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC9]])
+; AVX1-NEXT: [[STRIDED_VEC12:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC16]], 0
+; AVX1-NEXT: [[STRIDED_VEC11:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC16]], 1
; AVX1-NEXT: [[TMP36:%.*]] = sext <4 x i16> [[STRIDED_VEC5]] to <4 x i32>
; AVX1-NEXT: [[TMP37:%.*]] = sext <4 x i16> [[STRIDED_VEC6]] to <4 x i32>
; AVX1-NEXT: [[TMP38:%.*]] = sext <4 x i16> [[STRIDED_VEC7]] to <4 x i32>
@@ -214,17 +224,21 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX1-NEXT: [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[S2]], i64 [[TMP5]]
; AVX1-NEXT: [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[S2]], i64 [[TMP6]]
; AVX1-NEXT: [[WIDE_VEC13:%.*]] = load <8 x i16>, ptr [[TMP22]], align 2
-; AVX1-NEXT: [[STRIDED_VEC17:%.*]] = shufflevector <8 x i16> [[WIDE_VEC13]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC21:%.*]] = shufflevector <8 x i16> [[WIDE_VEC13]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC25:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC13]])
+; AVX1-NEXT: [[STRIDED_VEC17:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC25]], 0
+; AVX1-NEXT: [[STRIDED_VEC21:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC25]], 1
; AVX1-NEXT: [[WIDE_VEC14:%.*]] = load <8 x i16>, ptr [[TMP23]], align 2
-; AVX1-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <8 x i16> [[WIDE_VEC14]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC22:%.*]] = shufflevector <8 x i16> [[WIDE_VEC14]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC26:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC14]])
+; AVX1-NEXT: [[STRIDED_VEC18:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC26]], 0
+; AVX1-NEXT: [[STRIDED_VEC22:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC26]], 1
; AVX1-NEXT: [[WIDE_VEC18:%.*]] = load <8 x i16>, ptr [[TMP17]], align 2
-; AVX1-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <8 x i16> [[WIDE_VEC18]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <8 x i16> [[WIDE_VEC18]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC13:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC18]])
+; AVX1-NEXT: [[STRIDED_VEC19:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC13]], 0
+; AVX1-NEXT: [[STRIDED_VEC20:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC13]], 1
; AVX1-NEXT: [[WIDE_VEC21:%.*]] = load <8 x i16>, ptr [[TMP18]], align 2
-; AVX1-NEXT: [[STRIDED_VEC24:%.*]] = shufflevector <8 x i16> [[WIDE_VEC21]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC23:%.*]] = shufflevector <8 x i16> [[WIDE_VEC21]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC15:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC21]])
+; AVX1-NEXT: [[STRIDED_VEC24:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC15]], 0
+; AVX1-NEXT: [[STRIDED_VEC23:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC15]], 1
; AVX1-NEXT: [[TMP40:%.*]] = sext <4 x i16> [[STRIDED_VEC17]] to <4 x i32>
; AVX1-NEXT: [[TMP41:%.*]] = sext <4 x i16> [[STRIDED_VEC18]] to <4 x i32>
; AVX1-NEXT: [[TMP42:%.*]] = sext <4 x i16> [[STRIDED_VEC19]] to <4 x i32>
@@ -276,13 +290,15 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX1-NEXT: [[TMP48:%.*]] = shl nuw nsw i64 [[INDEX26]], 1
; AVX1-NEXT: [[TMP49:%.*]] = getelementptr inbounds i16, ptr [[S1]], i64 [[TMP48]]
; AVX1-NEXT: [[WIDE_VEC27:%.*]] = load <8 x i16>, ptr [[TMP49]], align 2
-; AVX1-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <8 x i16> [[WIDE_VEC27]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <8 x i16> [[WIDE_VEC27]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC27:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC27]])
+; AVX1-NEXT: [[STRIDED_VEC28:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC27]], 0
+; AVX1-NEXT: [[STRIDED_VEC29:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC27]], 1
; AVX1-NEXT: [[TMP50:%.*]] = sext <4 x i16> [[STRIDED_VEC28]] to <4 x i32>
; AVX1-NEXT: [[TMP51:%.*]] = getelementptr inbounds i16, ptr [[S2]], i64 [[TMP48]]
; AVX1-NEXT: [[WIDE_VEC30:%.*]] = load <8 x i16>, ptr [[TMP51]], align 2
-; AVX1-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <8 x i16> [[WIDE_VEC30]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX1-NEXT: [[STRIDED_VEC32:%.*]] = shufflevector <8 x i16> [[WIDE_VEC30]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT: [[STRIDED_VEC30:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC30]])
+; AVX1-NEXT: [[STRIDED_VEC31:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC30]], 0
+; AVX1-NEXT: [[STRIDED_VEC32:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC30]], 1
; AVX1-NEXT: [[TMP52:%.*]] = sext <4 x i16> [[STRIDED_VEC31]] to <4 x i32>
; AVX1-NEXT: [[TMP53:%.*]] = mul nsw <4 x i32> [[TMP52]], [[TMP50]]
; AVX1-NEXT: [[TMP54:%.*]] = sext <4 x i16> [[STRIDED_VEC29]] to <4 x i32>
@@ -346,13 +362,15 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
; AVX2-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; AVX2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[S1:%.*]], i64 [[TMP1]]
; AVX2-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP2]], align 2
-; AVX2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC5:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_VEC]])
+; AVX2-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC5]], 0
+; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC5]], 1
; AVX2-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[STRIDED_VEC]] to <8 x i32>
; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[S2:%.*]], i64 [[TMP1]]
; AVX2-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
-; AVX2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_VEC2]])
+; AVX2-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC2]], 0
+; AVX2-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC2]], 1
; AVX2-NEXT: [[TMP7:%.*]] = sext <8 x i16> [[STRIDED_VEC3]] to <8 x i32>
; AVX2-NEXT: [[TMP8:%.*]] = mul nsw <8 x i32> [[TMP7]], [[TMP4]]
; AVX2-NEXT: [[TMP9:%.*]] = sext <8 x i16> [[STRIDED_VEC1]] to <8 x i32>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
index 00fe1410d9592..ce05e7cad006d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
@@ -19,7 +19,8 @@ define void @pr56319(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [3 x i8], ptr [[SRC:%.*]], i64 [[INDEX]], i64 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <96 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <96 x i8> [[WIDE_VEC]], <96 x i8> poison, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <32 x i8>, <32 x i8>, <32 x i8> } @llvm.vector.deinterleave3.v96i8(<96 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <32 x i8>, <32 x i8>, <32 x i8> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <32 x i8> [[STRIDED_VEC]], ptr [[TMP3]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
@@ -28,7 +29,7 @@ define void @pr56319(ptr noalias %src, ptr noalias %dst) {
; CHECK: middle.block:
; CHECK-NEXT: br label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 32, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
@@ -36,12 +37,13 @@ define void @pr56319(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [3 x i8], ptr [[SRC]], i64 [[INDEX1]], i64 0
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <6 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <6 x i8> [[WIDE_VEC2]], <6 x i8> poison, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = call { <2 x i8>, <2 x i8>, <2 x i8> } @llvm.vector.deinterleave3.v6i8(<6 x i8> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x i8>, <2 x i8>, <2 x i8> } [[STRIDED_VEC4]], 0
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX1]]
; CHECK-NEXT: store <2 x i8> [[STRIDED_VEC3]], ptr [[TMP9]], align 1
; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 36
-; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
@@ -55,7 +57,7 @@ define void @pr56319(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 37
-; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 576a27bce5df4..0939841f1f76c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -511,8 +511,9 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP18:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc <8 x i32> [[TMP18]] to <8 x i8>
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[OFFSET_IDX]]
@@ -589,8 +590,9 @@ define void @test(ptr %A, ptr noalias %B) #0 {
; MAX-BW-NEXT: [[TMP31:%.*]] = add i64 [[OFFSET_IDX]], 62
; MAX-BW-NEXT: [[TMP32:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[OFFSET_IDX]]
; MAX-BW-NEXT: [[WIDE_VEC:%.*]] = load <64 x i32>, ptr [[TMP32]], align 4
-; MAX-BW-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
-; MAX-BW-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <64 x i32> [[WIDE_VEC]], <64 x i32> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
+; MAX-BW-NEXT: [[STRIDED_VEC2:%.*]] = call { <32 x i32>, <32 x i32> } @llvm.vector.deinterleave2.v64i32(<64 x i32> [[WIDE_VEC]])
+; MAX-BW-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <32 x i32>, <32 x i32> } [[STRIDED_VEC2]], 0
+; MAX-BW-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <32 x i32>, <32 x i32> } [[STRIDED_VEC2]], 1
; MAX-BW-NEXT: [[TMP33:%.*]] = add <32 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; MAX-BW-NEXT: [[TMP34:%.*]] = trunc <32 x i32> [[TMP33]] to <32 x i8>
; MAX-BW-NEXT: [[TMP35:%.*]] = getelementptr inbounds [1024 x i8], ptr [[B]], i64 0, i64 [[OFFSET_IDX]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll
index 8e196827471f5..02301b21767bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll
@@ -18,9 +18,10 @@ define void @load_store_interleave_group_with_gaps(ptr noalias %data, i64 nounde
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], <4 x i64> [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP2]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i64>, <4 x i64>, <4 x i64>, <4 x i64> } @llvm.vector.deinterleave4.v16i64(<16 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 1
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 3
; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[STRIDED_VEC]], <4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP3:%.*]] = or disjoint <4 x i64> [[TMP0]], splat (i64 1)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], <4 x i64> [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
index 8653d002e4cad..adb5f3c7c5a64 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
@@ -159,12 +159,12 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP6]], <4 x i64> [[TMP9]])
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -237,12 +237,12 @@ define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr no
; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP8]], <4 x i64> [[TMP9]])
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -315,12 +315,12 @@ define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP9]], <4 x i64> [[TMP6]])
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -393,12 +393,12 @@ define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %fa
; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP8]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP7]], <4 x i64> [[TMP6]])
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -477,12 +477,12 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[TMP5]], 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP7]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP8]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i64>, <4 x i64> } @llvm.vector.deinterleave2.v8i64(<8 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD2]], [[STRIDED_VEC]]
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> [[TMP6]], <4 x i64> [[TMP12]])
; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP16]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -557,16 +557,14 @@ define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP3]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i64>, <4 x i64>, <4 x i64> } @llvm.vector.deinterleave3.v12i64(<12 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 1
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i64>, <4 x i64>, <4 x i64> } [[STRIDED_VEC3]], 2
; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> [[TMP11]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP12]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i64> @llvm.vector.interleave3.v12i64(<4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <4 x i64> [[TMP8]])
; CHECK-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -688,19 +686,18 @@ define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV]], i32 0
; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <24 x i32>, ptr [[TMP5]], align 8
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <8 x i32>, <8 x i32>, <8 x i32> } @llvm.vector.deinterleave3.v24i32(<24 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC5]], 1
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <8 x i32>, <8 x i32>, <8 x i32> } [[STRIDED_VEC5]], 2
; CHECK-NEXT: [[TMP7:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC2]]
; CHECK-NEXT: [[TMP8:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC3]]
; CHECK-NEXT: [[TMP10:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC4]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> [[TMP13]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP14]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> [[TMP10]])
; CHECK-NEXT: store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index ead2f8481e539..008693caf9158 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -18,9 +18,10 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
; CHECK-NEXT: store i8 [[TMP3]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave6.v48i8(<8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> poison, <8 x i8> poison)
+; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <48 x i1> @llvm.vector.interleave6.v48i1(<8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]])
; CHECK-NEXT: [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
-; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i16> [[VEC_IND]], splat (i16 8)
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
@@ -70,12 +71,14 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: store i8 [[TMP5]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP4]]
-; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave6.v48i8(<8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> poison, <8 x i8> poison)
+; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <48 x i1> @llvm.vector.interleave6.v48i1(<8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]], <8 x i1> [[TMP1]])
; CHECK-NEXT: [[TMP8:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
-; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP6]], <48 x i1> [[TMP8]]), !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT: [[INTERLEAVED_MASK1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <48 x i1> [[TMP8]]), !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <48 x i8> @llvm.vector.interleave6.v48i8(<8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> zeroinitializer, <8 x i8> poison, <8 x i8> poison, <8 x i8> poison)
+; CHECK-NEXT: [[INTERLEAVED_MASK1:%.*]] = call <48 x i1> @llvm.vector.interleave6.v48i1(<8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <8 x i1> [[TMP2]], <8 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP9:%.*]] = and <48 x i1> [[INTERLEAVED_MASK1]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
-; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP7]], <48 x i1> [[TMP9]]), !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> [[INTERLEAVED_VEC1]], ptr align 1 [[TMP7]], <48 x i1> [[TMP9]]), !llvm.access.group [[ACC_GRP0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i64> [[STEP_ADD]], splat (i64 8)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10016
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll b/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
index b23186cd01275..3b2c51dc6d8f1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
@@ -28,7 +28,8 @@ define void @test(ptr %A) {
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[STRIDED_VEC]], splat (i32 2)
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP13]], i64 0
; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vplan-single-bit-ind-var-width-4.ll b/llvm/test/Transforms/LoopVectorize/X86/vplan-single-bit-ind-var-width-4.ll
index b0fabd240b181..84bd0b51506e8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vplan-single-bit-ind-var-width-4.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vplan-single-bit-ind-var-width-4.ll
@@ -23,10 +23,7 @@ define void @copy_bitcast_fusion(ptr noalias %foo, ptr noalias %bar) {
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP10]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP11]], i64 2
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[TMP12]], i64 3
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x float> [[TMP16]], <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x float> [[TMP17]], <8 x float> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef>, <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x float> [[TMP19]], <24 x float> poison, <24 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <24 x float> @llvm.vector.interleave6.v24f32(<4 x float> [[TMP16]], <4 x float> zeroinitializer, <4 x float> zeroinitializer, <4 x float> zeroinitializer, <4 x float> zeroinitializer, <4 x float> zeroinitializer)
; CHECK-NEXT: store <24 x float> [[INTERLEAVED_VEC]], ptr [[BAR]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index acbdf269934df..6b354e08fa05f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -150,9 +150,10 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP3]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -335,10 +336,11 @@ define void @masked_strided1_optsize(ptr noalias nocapture readonly %p, ptr noal
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP3]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -548,10 +550,11 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP3]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP2]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[TMP5]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -979,7 +982,8 @@ define void @unconditional_strided1_optsize(ptr noalias nocapture readonly %p, p
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP0]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: store <8 x i8> [[STRIDED_VEC]], ptr [[TMP2]], align 1
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -1164,10 +1168,11 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP3]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
@@ -1554,17 +1559,18 @@ define void @masked_strided2(ptr noalias nocapture readonly %p, ptr noalias noca
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC2]], 0
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC2]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp slt <8 x i8> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP8]], <8 x i8> [[STRIDED_VEC1]], <8 x i8> [[STRIDED_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[Q]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP3]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> [[TMP3]], <8 x i8> [[TMP5]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK1:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK1]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
@@ -2645,17 +2651,18 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP3]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP2]], <8 x i1> [[TMP2]])
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 0
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC1]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = icmp slt <8 x i8> [[STRIDED_VEC]], [[STRIDED_VEC3]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = select <8 x i1> [[TMP10]], <8 x i8> [[STRIDED_VEC3]], <8 x i8> [[STRIDED_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[TMP3]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = sub <8 x i8> zeroinitializer, [[TMP5]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP5]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> [[TMP5]], <8 x i8> [[TMP7]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK3:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP2]], <8 x i1> [[TMP2]])
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK3]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -3070,17 +3077,18 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[WIDE_MASKED_VEC]])
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC2]], 0
+; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[STRIDED_VEC2]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp slt <8 x i8> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP8]], <8 x i8> [[STRIDED_VEC1]], <8 x i8> [[STRIDED_VEC]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = sub <8 x i8> zeroinitializer, [[TMP3]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> [[TMP3]], <8 x i8> [[TMP5]])
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK1:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP0]], <8 x i1> [[TMP0]])
+; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK1]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index ce1b579dba671..818eef231c967 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -88,7 +88,7 @@ define void @test1(ptr noalias nocapture %points, ptr noalias nocapture readonly
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[POINTS]], i64 [[DOTIDX]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i16> @llvm.vector.interleave4.v16i16(<4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <4 x i16> poison, <4 x i16> poison)
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP1]], <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -260,8 +260,8 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[POINTS]], i64 [[DOTIDX]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i8], ptr [[Y]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[TMP0]], <4 x i16> poison)
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD1]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
-; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i16> @llvm.vector.interleave4.v16i16(<4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD1]], <4 x i16> poison, <4 x i16> poison)
+; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave4.v16i1(<4 x i1> [[TMP0]], <4 x i1> [[TMP0]], <4 x i1> [[TMP0]], <4 x i1> [[TMP0]])
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP2]], <16 x i1> [[TMP4]])
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index 50b72c49bf6bf..bdf441a8aea6a 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -312,8 +312,9 @@ define i32 @interleaved_access_forward(ptr %p, i64 %n) {
; INTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 8
-; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; INTER-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; INTER-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; INTER-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; INTER-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; INTER-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; INTER-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[TMP1]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -463,9 +464,10 @@ define i32 @interleaved_access_reverse(ptr %p, i64 %n) {
; INTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[OFFSET_IDX]], i32 0
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 -6
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 8
-; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; INTER-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; INTER-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
; INTER-NEXT: [[VEC_PHI:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; INTER-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; INTER-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; INTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; INTER-NEXT: [[TMP7:%.*]] = add <4 x i32> [[VEC_PHI]], [[TMP6]]
; INTER-NEXT: [[TMP10]] = add <4 x i32> [[VEC_PHI1]], [[TMP7]]
@@ -622,7 +624,8 @@ define void @predicated_store(ptr %p, i32 %x, i64 %n) {
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; INTER-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 8
-; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; INTER-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; INTER-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; INTER-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; INTER-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; INTER-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
@@ -1055,12 +1058,14 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
; INTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[NEXT_GEP]], align 8
-; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; INTER-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; INTER-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC]])
+; INTER-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
+; INTER-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 1
; INTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 4
; INTER-NEXT: [[WIDE_VEC5:%.*]] = load <16 x i32>, ptr [[TMP7]], align 8
-; INTER-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; INTER-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; INTER-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC5]])
+; INTER-NEXT: [[STRIDED_VEC6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 0
+; INTER-NEXT: [[TMP28:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 1
; INTER-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP18:%.*]] = sub <4 x i32> [[STRIDED_VEC]], [[STRIDED_VEC4]]
; INTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 2
@@ -1094,11 +1099,11 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: br label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; INTER-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; INTER-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ]
+; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP00:%.*]] = load i32, ptr [[P]], align 8
; INTER-NEXT: [[UNNAMEDTMP03:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
; INTER-NEXT: [[UNNAMEDTMP04:%.*]] = load i32, ptr [[UNNAMEDTMP03]], align 8
@@ -1531,8 +1536,9 @@ define void @pointer_operand_geps_with_different_indexed_types(ptr %A, ptr %B, i
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1, !alias.scope [[META27:![0-9]+]]
-; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
-; INTER-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
+; INTER-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } @llvm.vector.deinterleave8.v32i8(<32 x i8> [[WIDE_VEC]])
+; INTER-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 0
+; INTER-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 3
; INTER-NEXT: [[TMP5:%.*]] = xor <4 x i8> [[STRIDED_VEC3]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
; INTER-NEXT: store <4 x i8> [[TMP5]], ptr [[TMP6]], align 1, !alias.scope [[META30:![0-9]+]], !noalias [[META27]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
index f7e0c86fe8f7b..d2833bc105978 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
@@ -71,10 +71,11 @@ define void @conv_interleaved_loads_load(ptr noalias %a, ptr readonly %b, ptr re
; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
; CHECK-GATHER-DISABLED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; CHECK-GATHER-DISABLED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i16], ptr [[C]], i64 [[INDEX]], i64 0
-; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <16 x i1> @llvm.vector.interleave2.v16i1(<8 x i1> [[TMP2]], <8 x i1> [[TMP2]])
; CHECK-GATHER-DISABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[TMP3]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i16> poison)
-; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC1:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[WIDE_MASKED_VEC]])
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 1
; CHECK-GATHER-DISABLED-NEXT: [[TMP4:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[STRIDED_VEC2]]
; CHECK-GATHER-DISABLED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP4]], <8 x i16> zeroinitializer
; CHECK-GATHER-DISABLED-NEXT: [[TMP5:%.*]] = add <8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 1683f784daf75..14961970884f7 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -1161,15 +1161,19 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
; INTERLEAVE-NEXT: [[TMP26:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
; INTERLEAVE-NEXT: [[TMP27:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP25]]
; INTERLEAVE-NEXT: [[WIDE_VEC2:%.*]] = load <32 x float>, ptr [[TMP26]], align 4
-; INTERLEAVE-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x float> [[WIDE_VEC2]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave8.v32f32(<32 x float> [[WIDE_VEC2]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 0
; INTERLEAVE-NEXT: [[WIDE_VEC3:%.*]] = load <32 x float>, ptr [[TMP27]], align 4
-; INTERLEAVE-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <32 x float> [[WIDE_VEC3]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+; INTERLEAVE-NEXT: [[STRIDED_VEC6:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave8.v32f32(<32 x float> [[WIDE_VEC3]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC6]], 0
; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[OFFSET_IDX]]
; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP25]]
; INTERLEAVE-NEXT: [[WIDE_VEC4:%.*]] = load <32 x float>, ptr [[TMP8]], align 4
-; INTERLEAVE-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <32 x float> [[WIDE_VEC4]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+; INTERLEAVE-NEXT: [[STRIDED_VEC8:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave8.v32f32(<32 x float> [[WIDE_VEC4]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC8]], 0
; INTERLEAVE-NEXT: [[WIDE_VEC6:%.*]] = load <32 x float>, ptr [[TMP9]], align 4
-; INTERLEAVE-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <32 x float> [[WIDE_VEC6]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+; INTERLEAVE-NEXT: [[STRIDED_VEC9:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave8.v32f32(<32 x float> [[WIDE_VEC6]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC9]], 0
; INTERLEAVE-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[VEC_PHI]], splat (float 1.000000e+00)
; INTERLEAVE-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], splat (float 1.000000e+00)
; INTERLEAVE-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[TMP10]], [[STRIDED_VEC3]]
@@ -1476,9 +1480,11 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) {
; INTERLEAVE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP8]], i32 1
; INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP30]], i32 1
; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 8
-; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; INTERLEAVE-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; INTERLEAVE-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP13]], align 8
-; INTERLEAVE-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; INTERLEAVE-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC1]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC3]], 0
; INTERLEAVE-NEXT: [[TMP17:%.*]] = xor <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; INTERLEAVE-NEXT: [[TMP18:%.*]] = xor <4 x i32> [[STRIDED_VEC2]], [[BROADCAST_SPLAT]]
; INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <4 x i32> [[TMP17]], i64 0
@@ -1867,9 +1873,11 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP18]]
; INTERLEAVE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP20]], align 1, !alias.scope [[META17:![0-9]+]]
-; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; INTERLEAVE-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; INTERLEAVE-NEXT: [[WIDE_VEC3:%.*]] = load <16 x i32>, ptr [[TMP21]], align 1, !alias.scope [[META17]]
-; INTERLEAVE-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC3]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; INTERLEAVE-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC3]])
+; INTERLEAVE-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 0
; INTERLEAVE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1
; INTERLEAVE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1
; INTERLEAVE-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP12]], i32 1
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
index 2d29e55e0d1a8..2b2c3a5d00327 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
@@ -37,8 +37,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; CHECK: vector.body:
; CHECK: %wide.vec = load <8 x i32>, ptr {{.*}}, align 4
-; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %wide.vec)
%class.Complex = type { float, float }
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
index 4c3e85f0e1c71..3da0ba7d3696b 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
@@ -42,7 +42,8 @@ define void @_Z4testPfS_m(ptr noalias nocapture %out, ptr noalias nocapture read
; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[IN]], i64 [[TMP6]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
index 32a5b38651d27..2815666149776 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
@@ -21,9 +21,10 @@ define void @dead_member_order(ptr noalias %src, ptr noalias %dst1, ptr noalias
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [3 x i8], ptr [[SRC]], i32 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i8>, <4 x i8>, <4 x i8> } @llvm.vector.deinterleave3.v12i8(<12 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST1]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[STRIDED_VEC1]], ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
@@ -87,8 +88,9 @@ define void @dead_member_metadata(ptr noalias %src, ptr noalias %dst, i32 %N) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [2 x i8], ptr [[SRC]], i32 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i8>, <4 x i8> } @llvm.vector.deinterleave2.v8i8(<8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[STRIDED_VEC1]], ptr [[TMP3]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
index aabfbf58adba8..1b92d47e907ac 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
@@ -17,8 +17,9 @@ define void @gep_for_first_member_does_not_dominate_insert_point(ptr %str, ptr n
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[STR]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i8>, <4 x i8> } @llvm.vector.deinterleave2.v8i8(<8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i8> [[STRIDED_VEC2]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP5]], ptr [[TMP6]], align 1
@@ -71,8 +72,9 @@ define void @test_ig_insert_pos_at_end_of_vpbb(ptr noalias %dst, ptr noalias %sr
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr nusw { i16, i16, i16, i16 }, ptr [[SRC]], i64 [[TMP3]], i32 2
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr nusw i8, ptr [[TMP4]], i32 -4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.vector.deinterleave4.v16i16(<16 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } [[STRIDED_VEC1]], 2
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[STRIDED_VEC]], i64 3
; CHECK-NEXT: store i16 [[TMP6]], ptr [[DST]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
@@ -161,8 +163,9 @@ define i64 @interleave_group_load_pointer_type(ptr %start, ptr %end) {
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 16
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 -8
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x ptr>, ptr [[TMP8]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x ptr> [[WIDE_VEC]], <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x ptr> [[WIDE_VEC]], <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x ptr>, <4 x ptr>, <4 x ptr> } @llvm.vector.deinterleave3.v12p0(<12 x ptr> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x ptr>, <4 x ptr>, <4 x ptr> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x ptr>, <4 x ptr>, <4 x ptr> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint <4 x ptr> [[STRIDED_VEC3]] to <4 x i64>
; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint <4 x ptr> [[STRIDED_VEC]] to <4 x i64>
; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i64> [[TMP9]], [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll
index e7f95f5237537..5ea610706dfa3 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll
@@ -17,9 +17,10 @@ define void @nusw_preservation(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nusw [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr nusw i32, ptr [[TMP0]], i64 -6
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
@@ -27,8 +28,7 @@ define void @nusw_preservation(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr nusw i32, ptr [[TMP5]], i64 -6
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
@@ -77,9 +77,10 @@ define void @inbounds_preservation(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 -6
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
@@ -87,8 +88,7 @@ define void @inbounds_preservation(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -6
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
@@ -137,9 +137,10 @@ define void @nuw_drop(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nuw [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[TMP0]], i64 -6
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
@@ -147,8 +148,7 @@ define void @nuw_drop(ptr noalias %A, ptr %B) {
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP5]], i64 -6
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
@@ -197,8 +197,9 @@ define void @nusw_preservation_2(ptr %src, ptr noalias %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nusw i8, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr nusw i8, ptr [[TMP1]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i8>, <4 x i8> } @llvm.vector.deinterleave2.v8i8(<8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr nusw i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
@@ -247,8 +248,9 @@ define void @inbounds_preservation_2(ptr %src, ptr noalias %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i8>, <4 x i8> } @llvm.vector.deinterleave2.v8i8(<8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
@@ -297,8 +299,9 @@ define void @nuw_drop_2(ptr %src, ptr noalias %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i8>, <4 x i8> } @llvm.vector.deinterleave2.v8i8(<8 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i8>, <4 x i8> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr nuw i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
index 3067970b929b9..1416a8915df20 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
@@ -36,8 +36,7 @@ define void @merge_tbaa_interleave_group(ptr nocapture readonly %p, ptr noalias
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> [[TMP14]], double [[TMP13]], i64 1
; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP15]], splat (double 3.000000e+00)
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[TMP8]], <2 x double> [[TMP16]])
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP9]], align 8, !tbaa [[TBAA6:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
@@ -109,12 +108,12 @@ define void @ir_tbaa_different(ptr %base, ptr %end, ptr %src) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x float> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> [[TMP6]], <2 x float> [[TMP7]])
; CHECK-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META13]], !noalias [[META10]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -186,12 +185,12 @@ define void @noalias_metadata_from_versioning(ptr %base, ptr %end, ptr %src) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <2 x float>, <2 x float> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[STRIDED_VEC]], splat (float 1.000000e+01)
; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x float> [[STRIDED_VEC3]], splat (float 1.000000e+01)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> [[TMP6]], <2 x float> [[TMP7]])
; CHECK-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
index 6bea5ec771c0e..ad1d010331a33 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
@@ -32,7 +32,8 @@ define void @interleaved_with_cond_store_0(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
@@ -129,7 +130,8 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP18]], i32 1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
@@ -147,7 +149,8 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]
; CHECK: pred.store.continue2:
; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x i64> [[WIDE_VEC3]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC3]])
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC5]], 0
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[STRIDED_VEC4]], i64 0
; CHECK-NEXT: store i64 [[TMP11]], ptr [[TMP2]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i64> [[STRIDED_VEC4]], i64 1
@@ -236,7 +239,8 @@ define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP15]], i32 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: store i64 [[X]], ptr [[TMP1]], align 8
; CHECK-NEXT: store i64 [[X]], ptr [[TMP2]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
index 5f52b29a00d98..5909d239193f2 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
@@ -60,7 +60,8 @@ define void @wrap_around_scev_check(ptr noalias %a, ptr noalias %b, i8 %x, i8 %y
; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[OFFSET_IDX]] to i64
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP13:%.*]] = shl <4 x i32> [[STRIDED_VEC]], splat (i32 1)
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP14]], align 4
@@ -204,7 +205,8 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP8]], 15
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index 7de0f1a53b448..94b2ce6963a32 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -36,13 +36,13 @@ define void @test_array_load2_store2(i32 %C, i32 %D) {
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 [[TMP1]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[BROADCAST_SPLAT3]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 [[TMP1]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
@@ -106,17 +106,15 @@ define void @test_struct_array_load3_store3() {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr @A, i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 2
; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [1024 x [[STRUCT_ST3:%.*]]], ptr @S, i64 0, i64 [[INDEX]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[STRIDED_VEC2]], splat (i32 2)
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], splat (i32 3)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP5]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i32> [[TMP7]], <12 x i32> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -188,10 +186,11 @@ define i32 @test_struct_load4(ptr nocapture readonly %S) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ST4:%.*]], ptr [[S:%.*]], i64 [[INDEX]], i32 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 1
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 2
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC4]], 3
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP2:%.*]] = sub <4 x i32> [[TMP1]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP2]], [[STRIDED_VEC2]]
@@ -261,10 +260,7 @@ define void @test_struct_store4(ptr noalias nocapture readonly %A, ptr noalias n
; CHECK-NEXT: [[TMP1:%.*]] = shl nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 3)
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 4)
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[TMP8]], <16 x i32> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
; CHECK-NEXT: store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -335,9 +331,10 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 -6
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
@@ -345,8 +342,7 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 -6
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
@@ -403,7 +399,8 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP1:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
; CHECK-NEXT: [[TMP3:%.*]] = lshr exact i64 [[TMP2]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP3]]
@@ -422,8 +419,8 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[UNNAMEDTMP1]]
; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 1024
@@ -480,7 +477,8 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
; CHECK-NEXT: [[TMP7:%.*]] = shl nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP7]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP6:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = lshr exact i64 [[TMP7]], 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP10]]
@@ -500,8 +498,8 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[UNNAMEDTMP1]]
; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N1]]
@@ -645,16 +643,17 @@ define void @mixed_load2_store2(ptr noalias nocapture readonly %A, ptr noalias n
; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC5]], 1
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC2]])
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC2]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
@@ -713,16 +712,14 @@ define void @mixed_load3_store3(ptr nocapture %A) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 2
; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_IND]]
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_IND]]
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[STRIDED_VEC3]], [[VEC_IND]]
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i32> [[TMP6]], <12 x i32> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -799,8 +796,9 @@ define void @int_float_struct(ptr nocapture readonly %A) {
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_INTFLOAT:%.*]], ptr [[A:%.*]], i64 [[INDEX]], i32 0
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 1
; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[STRIDED_VEC2]] to <4 x float>
; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI1]]
; CHECK-NEXT: [[TMP3]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP1]]
@@ -888,7 +886,8 @@ define void @PR27626_0(ptr %p, i32 %z, i64 %n) {
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 0
; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 1
@@ -972,7 +971,8 @@ define i32 @PR27626_1(ptr %p, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]], i32 1
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP20]], i32 1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 0
; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 1
@@ -982,7 +982,8 @@ define i32 @PR27626_1(ptr %p, i64 %n) {
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 3
; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC3]], 0
; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1074,7 +1075,8 @@ define void @PR27626_2(ptr %p, i64 %n, i32 %z) {
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 0
; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 1
@@ -1165,7 +1167,8 @@ define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) {
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC1]], 0
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 0
; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 1
@@ -1175,7 +1178,8 @@ define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) {
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 3
; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC3]], 0
; CHECK-NEXT: [[TMP17]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
@@ -1274,8 +1278,7 @@ define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) {
; CHECK-NEXT: store i32 [[X]], ptr [[TMP9]], align 4
; CHECK-NEXT: store i32 [[X]], ptr [[TMP10]], align 4
; CHECK-NEXT: store i32 [[X]], ptr [[TMP14]], align 4
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP15]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]])
; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1478,8 +1481,9 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw i64 [[TMP7]], 1
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP10]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, ptr [[TMP14]], align 4, !alias.scope [[META27:![0-9]+]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[STRIDED_VEC4]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[STRIDED_VEC4]] = extractvalue { <4 x i16>, <4 x i16> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32>
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP19:%.*]] = sext <4 x i16> [[TMP17]] to <4 x i32>
diff --git a/llvm/test/Transforms/LoopVectorize/pr128062-interleaved-accesses-narrow-group.ll b/llvm/test/Transforms/LoopVectorize/pr128062-interleaved-accesses-narrow-group.ll
index 5c38d670734fd..7ad26ec133b9b 100644
--- a/llvm/test/Transforms/LoopVectorize/pr128062-interleaved-accesses-narrow-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr128062-interleaved-accesses-narrow-group.ll
@@ -94,10 +94,11 @@ define void @opcode_mismatch(ptr %dst.start, i8 %a, i16 %b) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST_START]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_VEC]], <16 x i8> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_VEC]], <16 x i8> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i8> [[WIDE_VEC]], <16 x i8> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <16 x i8> [[WIDE_VEC]], <16 x i8> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = call { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } @llvm.vector.deinterleave4.v16i8(<16 x i8> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 2
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8> } [[STRIDED_VEC1]], 3
; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i8> [[STRIDED_VEC]] to <4 x i16>
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw <4 x i16> [[TMP0]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i16> [[TMP1]], splat (i16 255)
@@ -118,10 +119,7 @@ define void @opcode_mismatch(ptr %dst.start, i8 %a, i16 %b) {
; CHECK-NEXT: [[TMP17:%.*]] = udiv <4 x i16> [[TMP16]], splat (i16 255)
; CHECK-NEXT: [[TMP18:%.*]] = trunc nuw <4 x i16> [[TMP17]] to <4 x i8>
; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i8> [[BROADCAST_SPLAT2]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP14]], <4 x i8> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <8 x i8> [[TMP20]], <8 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <16 x i8> @llvm.vector.interleave4.v16i8(<4 x i8> [[TMP4]], <4 x i8> [[TMP9]], <4 x i8> [[TMP14]], <4 x i8> [[TMP19]])
; CHECK-NEXT: store <16 x i8> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
diff --git a/llvm/test/Transforms/LoopVectorize/pr31098.ll b/llvm/test/Transforms/LoopVectorize/pr31098.ll
index f183e30a04d29..c3bc2d1b9e54f 100644
--- a/llvm/test/Transforms/LoopVectorize/pr31098.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr31098.ll
@@ -56,7 +56,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; CHECK-LABEL: Test
; CHECK: LAA: No unsafe dependent memory operations in loop. We don't need runtime memory checks.
; CHECK: vector.body:
-; CHECK: <4 x i32>
+; CHECK: call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
%class.Complex = type { float, float }
diff --git a/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll b/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
index 4013ee52bb5e8..74b17df188ffc 100644
--- a/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
@@ -18,8 +18,9 @@ define void @f(ptr noalias %p, ptr noalias %q, ptr noalias %dst, i64 %stride) {
; VF4UF1-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF4UF1-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
; VF4UF1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
-; VF4UF1-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; VF4UF1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4UF1-NEXT: [[STRIDED_VEC:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[WIDE_VEC]])
+; VF4UF1-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 0
+; VF4UF1-NEXT: [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[STRIDED_VEC]], 1
; VF4UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF4UF1-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP2]], align 4
; VF4UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
@@ -135,11 +136,13 @@ define void @f(ptr noalias %p, ptr noalias %q, ptr noalias %dst, i64 %stride) {
; VF2UF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP1]]
; VF2UF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
; VF2UF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
-; VF2UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
-; VF2UF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; VF2UF2-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> [[WIDE_VEC]])
+; VF2UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC]], 0
+; VF2UF2-NEXT: [[TMP14:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC]], 1
; VF2UF2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
-; VF2UF2-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
-; VF2UF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; VF2UF2-NEXT: [[STRIDED_VEC2:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> [[WIDE_VEC2]])
+; VF2UF2-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 0
+; VF2UF2-NEXT: [[TMP15:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[STRIDED_VEC2]], 1
; VF2UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF2UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 2
; VF2UF2-NEXT: store <2 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP5]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
index 53ea8530047f5..4bc3dc6ab1e6a 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
@@ -41,10 +41,12 @@ define void @hoist_invariant_load(ptr %invariant_ptr, i64 %num_elements, ptr %ar
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 64
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [32 x i8], ptr [[ARRAY]], i64 [[I2]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 96
-; CHECK-NEXT: [[TMP12:%.*]] = load <6 x double>, ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[TMP12]], <6 x double> poison, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT: [[TMP13:%.*]] = load <6 x double>, ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
-; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <6 x double> [[TMP13]], <6 x double> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = tail call { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave4.v8f64(<8 x double> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <8 x double>, ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = tail call { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave4.v8f64(<8 x double> [[WIDE_VEC4]])
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } [[STRIDED_VEC6]], 0
; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP17:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC5]]
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP14]], i64 0
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
index 096581961b79a..c8644df6728ed 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
@@ -16,20 +16,44 @@ define void @same_op2(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[WIDE_VEC15:%.*]] = load <8 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[STRIDED_VEC15:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC15]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC15]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC15]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC18:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC18]])
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC17]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC17]], 1
; CHECK-NEXT: [[WIDE_VEC21:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[STRIDED_VEC19:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC21]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC19]], 0
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC19]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = fmul fast <4 x float> [[TMP10]], [[TMP8]]
+; CHECK-NEXT: [[TMP15:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP17]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC24]])
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC21]], 0
+; CHECK-NEXT: [[TMP19:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC21]], 1
; CHECK-NEXT: [[WIDE_VEC27:%.*]] = load <8 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = fmul fast <8 x float> [[WIDE_VEC18]], [[WIDE_VEC]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <8 x float> [[WIDE_VEC24]], [[TMP7]]
+; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC27]])
+; CHECK-NEXT: [[TMP20:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC23]], 0
+; CHECK-NEXT: [[TMP21:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC23]], 1
+; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP18]], [[TMP14]]
+; CHECK-NEXT: [[TMP23:%.*]] = fadd fast <4 x float> [[TMP20]], [[TMP15]]
+; CHECK-NEXT: [[TMP24:%.*]] = fmul fast <4 x float> [[TMP11]], [[TMP16]]
+; CHECK-NEXT: [[TMP25:%.*]] = fmul fast <4 x float> [[TMP13]], [[TMP7]]
+; CHECK-NEXT: [[TMP26:%.*]] = fadd fast <4 x float> [[TMP19]], [[TMP24]]
+; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP25]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP22]], <4 x float> [[TMP26]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x float> [[WIDE_VEC21]], [[WIDE_VEC15]]
-; CHECK-NEXT: [[INTERLEAVED_VEC30:%.*]] = fadd fast <8 x float> [[WIDE_VEC27]], [[TMP8]]
+; CHECK-NEXT: [[INTERLEAVED_VEC30:%.*]] = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP23]], <4 x float> [[TMP27]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC30]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 576
@@ -118,8 +142,7 @@ define void @same_op2_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[C]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -128,16 +151,34 @@ define void @same_op2_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC]], 1
; CHECK-NEXT: [[WIDE_VEC13:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[STRIDED_VEC13:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC13]])
+; CHECK-NEXT: [[TMP23:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC13]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC13]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <4 x float> [[TMP11]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP24:%.*]] = fmul fast <4 x float> [[TMP23]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_VEC16:%.*]] = load <8 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[STRIDED_VEC15:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC16]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC15]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC15]], 1
; CHECK-NEXT: [[WIDE_VEC19:%.*]] = load <8 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x float> [[WIDE_VEC]], [[TMP1]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <8 x float> [[WIDE_VEC16]], [[TMP8]]
+; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE_VEC19]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC17]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <4 x float>, <4 x float> } [[STRIDED_VEC17]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = fadd fast <4 x float> [[TMP13]], [[TMP9]]
+; CHECK-NEXT: [[TMP18:%.*]] = fadd fast <4 x float> [[TMP15]], [[TMP24]]
+; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x float> [[TMP12]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <4 x float> [[TMP8]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x float> [[TMP14]], [[TMP19]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP16]], [[TMP20]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP17]], <4 x float> [[TMP21]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <8 x float> [[WIDE_VEC13]], [[TMP2]]
-; CHECK-NEXT: [[INTERLEAVED_VEC22:%.*]] = fadd fast <8 x float> [[WIDE_VEC19]], [[TMP9]]
+; CHECK-NEXT: [[INTERLEAVED_VEC22:%.*]] = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[TMP18]], <4 x float> [[TMP22]])
; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC22]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 576
@@ -226,12 +267,29 @@ define void @same_op3(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul nuw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 2
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC16:%.*]] = load <12 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[STRIDED_VEC15:%.*]] = tail call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC16]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC15]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC15]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC15]], 2
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <4 x float> [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC20:%.*]] = load <12 x float>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <12 x float> [[WIDE_VEC16]], [[WIDE_VEC]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <12 x float> [[WIDE_VEC20]], [[TMP3]]
+; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = tail call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC20]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC17]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC17]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC17]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[TMP11]], [[TMP9]]
+; CHECK-NEXT: [[TMP15:%.*]] = fmul fast <4 x float> [[TMP7]], [[TMP3]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd fast <4 x float> [[TMP12]], [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP18:%.*]] = fadd fast <4 x float> [[TMP13]], [[TMP17]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> [[TMP14]], <4 x float> [[TMP16]], <4 x float> [[TMP18]])
; CHECK-NEXT: store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 384
@@ -320,17 +378,30 @@ define void @same_op3_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[C]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <12 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul nuw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[TMP7]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC14:%.*]] = load <12 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <12 x float> [[WIDE_VEC]], [[TMP2]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <12 x float> [[WIDE_VEC14]], [[TMP4]]
+; CHECK-NEXT: [[STRIDED_VEC13:%.*]] = tail call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE_VEC14]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC13]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC13]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[STRIDED_VEC13]], 2
+; CHECK-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[TMP8]], [[TMP6]]
+; CHECK-NEXT: [[TMP16:%.*]] = fmul fast <4 x float> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[TMP9]], [[TMP16]]
+; CHECK-NEXT: [[TMP14:%.*]] = fmul fast <4 x float> [[TMP5]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP15:%.*]] = fadd fast <4 x float> [[TMP10]], [[TMP14]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> [[TMP11]], <4 x float> [[TMP13]], <4 x float> [[TMP15]])
; CHECK-NEXT: store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 384
@@ -980,17 +1051,50 @@ define void @same_op8_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[C]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave8.v16f32(<16 x float> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP36:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 3
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 5
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 6
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC]], 7
+; CHECK-NEXT: [[TMP11:%.*]] = fmul fast <2 x float> [[TMP3]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC19:%.*]] = load <16 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <16 x float> [[WIDE_VEC]], [[TMP1]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <16 x float> [[WIDE_VEC19]], [[TMP4]]
+; CHECK-NEXT: [[STRIDED_VEC13:%.*]] = tail call { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave8.v16f32(<16 x float> [[WIDE_VEC19]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 3
+; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 5
+; CHECK-NEXT: [[TMP19:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 6
+; CHECK-NEXT: [[TMP20:%.*]] = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float>, <2 x float> } [[STRIDED_VEC13]], 7
+; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <2 x float> [[TMP13]], [[TMP11]]
+; CHECK-NEXT: [[TMP22:%.*]] = fmul fast <2 x float> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP23:%.*]] = fadd fast <2 x float> [[TMP14]], [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = fmul fast <2 x float> [[TMP12]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP37:%.*]] = fadd fast <2 x float> [[TMP15]], [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <2 x float> [[TMP36]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <2 x float> [[TMP16]], [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = fmul fast <2 x float> [[TMP7]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP29:%.*]] = fadd fast <2 x float> [[TMP17]], [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = fmul fast <2 x float> [[TMP8]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP31:%.*]] = fadd fast <2 x float> [[TMP18]], [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = fmul fast <2 x float> [[TMP9]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP33:%.*]] = fadd fast <2 x float> [[TMP19]], [[TMP32]]
+; CHECK-NEXT: [[TMP34:%.*]] = fmul fast <2 x float> [[TMP10]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP35:%.*]] = fadd fast <2 x float> [[TMP20]], [[TMP34]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <16 x float> @llvm.vector.interleave8.v16f32(<2 x float> [[TMP21]], <2 x float> [[TMP23]], <2 x float> [[TMP37]], <2 x float> [[TMP27]], <2 x float> [[TMP29]], <2 x float> [[TMP31]], <2 x float> [[TMP33]], <2 x float> [[TMP35]])
; CHECK-NEXT: store <16 x float> [[INTERLEAVED_VEC]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 144
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
index aed66ac06aa47..96e401cd995ad 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll
@@ -19,9 +19,23 @@ define void @add4(ptr noalias noundef %x, ptr noalias noundef %y, i32 noundef %n
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = add <32 x i16> [[WIDE_VEC24]], [[WIDE_VEC]]
+; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC24]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC22]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC22]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC22]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC22]], 3
+; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i16> [[TMP7]], [[TMP2]]
+; CHECK-NEXT: [[TMP12:%.*]] = add <8 x i16> [[TMP8]], [[TMP6]]
+; CHECK-NEXT: [[TMP13:%.*]] = add <8 x i16> [[TMP9]], [[TMP4]]
+; CHECK-NEXT: [[TMP14:%.*]] = add <8 x i16> [[TMP10]], [[TMP5]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP11]], <8 x i16> [[TMP12]], <8 x i16> [[TMP13]], <8 x i16> [[TMP14]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -139,23 +153,23 @@ define void @addsubs(ptr noalias noundef %x, ptr noundef %y, i32 noundef %n) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 2
+; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC26:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC27:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC24:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC24]])
+; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 0
+; CHECK-NEXT: [[STRIDED_VEC26:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 1
+; CHECK-NEXT: [[STRIDED_VEC27:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 2
+; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 3
; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[STRIDED_VEC25]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = sub <8 x i16> [[STRIDED_VEC26]], [[STRIDED_VEC21]]
; CHECK-NEXT: [[TMP5:%.*]] = add <8 x i16> [[STRIDED_VEC27]], [[STRIDED_VEC22]]
; CHECK-NEXT: [[TMP6:%.*]] = sub <8 x i16> [[STRIDED_VEC28]], [[STRIDED_VEC23]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> [[TMP8]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <8 x i16> [[TMP5]], <8 x i16> [[TMP6]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -273,23 +287,23 @@ define void @add2sub2(ptr noalias noundef %x, ptr noundef %y, i32 noundef %n) {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC21:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC22:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 2
+; CHECK-NEXT: [[STRIDED_VEC23:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC26:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC27:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <32 x i16> [[WIDE_VEC24]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC24:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC24]])
+; CHECK-NEXT: [[STRIDED_VEC25:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 0
+; CHECK-NEXT: [[STRIDED_VEC26:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 1
+; CHECK-NEXT: [[STRIDED_VEC27:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 2
+; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC24]], 3
; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[STRIDED_VEC25]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP4:%.*]] = add <8 x i16> [[STRIDED_VEC26]], [[STRIDED_VEC21]]
; CHECK-NEXT: [[TMP5:%.*]] = sub <8 x i16> [[STRIDED_VEC27]], [[STRIDED_VEC22]]
; CHECK-NEXT: [[TMP6:%.*]] = sub <8 x i16> [[STRIDED_VEC28]], [[STRIDED_VEC23]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> [[TMP8]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <8 x i16> [[TMP5]], <8 x i16> [[TMP6]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -407,12 +421,34 @@ define void @addmul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z, i32
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Z:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC31:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[STRIDED_VEC29:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC31]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC29]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC29]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC29]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC29]], 3
+; CHECK-NEXT: [[TMP11:%.*]] = mul <8 x i16> [[TMP7]], [[TMP6]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC36:%.*]] = load <32 x i16>, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[TMP4:%.*]] = mul <32 x i16> [[WIDE_VEC31]], [[WIDE_VEC]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = add <32 x i16> [[TMP4]], [[WIDE_VEC36]]
+; CHECK-NEXT: [[STRIDED_VEC31:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC36]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 3
+; CHECK-NEXT: [[TMP17:%.*]] = add <8 x i16> [[TMP11]], [[TMP13]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul <8 x i16> [[TMP8]], [[TMP3]]
+; CHECK-NEXT: [[TMP19:%.*]] = add <8 x i16> [[TMP18]], [[TMP14]]
+; CHECK-NEXT: [[TMP20:%.*]] = mul <8 x i16> [[TMP9]], [[TMP4]]
+; CHECK-NEXT: [[TMP21:%.*]] = add <8 x i16> [[TMP20]], [[TMP15]]
+; CHECK-NEXT: [[TMP22:%.*]] = mul <8 x i16> [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[TMP23:%.*]] = add <8 x i16> [[TMP22]], [[TMP16]]
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP17]], <8 x i16> [[TMP19]], <8 x i16> [[TMP21]], <8 x i16> [[TMP23]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -563,23 +599,26 @@ define void @addsubsmul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z,
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC30:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC29:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 2
+; CHECK-NEXT: [[STRIDED_VEC30:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Z:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC31:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[STRIDED_VEC32:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC35:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC36:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC31]])
+; CHECK-NEXT: [[STRIDED_VEC32:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 0
+; CHECK-NEXT: [[STRIDED_VEC33:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 1
+; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 2
+; CHECK-NEXT: [[STRIDED_VEC35:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 3
; CHECK-NEXT: [[TMP4:%.*]] = mul <8 x i16> [[STRIDED_VEC32]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC36:%.*]] = load <32 x i16>, ptr [[TMP3]], align 2
-; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC31:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC36]])
+; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 0
+; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 1
+; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 2
+; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 3
; CHECK-NEXT: [[TMP5:%.*]] = add <8 x i16> [[TMP4]], [[STRIDED_VEC37]]
; CHECK-NEXT: [[TMP6:%.*]] = mul <8 x i16> [[STRIDED_VEC33]], [[STRIDED_VEC28]]
; CHECK-NEXT: [[TMP7:%.*]] = sub <8 x i16> [[STRIDED_VEC38]], [[TMP6]]
@@ -587,9 +626,7 @@ define void @addsubsmul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z,
; CHECK-NEXT: [[TMP9:%.*]] = add <8 x i16> [[TMP8]], [[STRIDED_VEC39]]
; CHECK-NEXT: [[TMP10:%.*]] = mul <8 x i16> [[STRIDED_VEC35]], [[STRIDED_VEC30]]
; CHECK-NEXT: [[TMP11:%.*]] = sub <8 x i16> [[STRIDED_VEC40]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i16> [[TMP12]], <16 x i16> [[TMP13]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP5]], <8 x i16> [[TMP7]], <8 x i16> [[TMP9]], <8 x i16> [[TMP11]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
@@ -740,23 +777,26 @@ define void @add2sub2mul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z,
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Y:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <32 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC30:%.*]] = shufflevector <32 x i16> [[WIDE_VEC]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 0
+; CHECK-NEXT: [[STRIDED_VEC28:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 1
+; CHECK-NEXT: [[STRIDED_VEC29:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 2
+; CHECK-NEXT: [[STRIDED_VEC30:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC1]], 3
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[Z:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC31:%.*]] = load <32 x i16>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[STRIDED_VEC32:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC33:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC35:%.*]] = shufflevector <32 x i16> [[WIDE_VEC31]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC36:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC31]])
+; CHECK-NEXT: [[STRIDED_VEC32:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 0
+; CHECK-NEXT: [[STRIDED_VEC33:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 1
+; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 2
+; CHECK-NEXT: [[STRIDED_VEC35:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC36]], 3
; CHECK-NEXT: [[TMP4:%.*]] = mul <8 x i16> [[STRIDED_VEC32]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[X:%.*]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_VEC36:%.*]] = load <32 x i16>, ptr [[TMP3]], align 2
-; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = shufflevector <32 x i16> [[WIDE_VEC36]], <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT: [[STRIDED_VEC31:%.*]] = tail call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> [[WIDE_VEC36]])
+; CHECK-NEXT: [[STRIDED_VEC37:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 0
+; CHECK-NEXT: [[STRIDED_VEC38:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 1
+; CHECK-NEXT: [[STRIDED_VEC39:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 2
+; CHECK-NEXT: [[STRIDED_VEC40:%.*]] = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } [[STRIDED_VEC31]], 3
; CHECK-NEXT: [[TMP5:%.*]] = add <8 x i16> [[TMP4]], [[STRIDED_VEC37]]
; CHECK-NEXT: [[TMP6:%.*]] = mul <8 x i16> [[STRIDED_VEC33]], [[STRIDED_VEC28]]
; CHECK-NEXT: [[TMP7:%.*]] = add <8 x i16> [[TMP6]], [[STRIDED_VEC38]]
@@ -764,9 +804,7 @@ define void @add2sub2mul(ptr noalias noundef %x, ptr noundef %y, ptr noundef %z,
; CHECK-NEXT: [[TMP9:%.*]] = sub <8 x i16> [[STRIDED_VEC39]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = mul <8 x i16> [[STRIDED_VEC35]], [[STRIDED_VEC30]]
; CHECK-NEXT: [[TMP11:%.*]] = sub <8 x i16> [[STRIDED_VEC40]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i16> [[TMP12]], <16 x i16> [[TMP13]], <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[TMP5]], <8 x i16> [[TMP7]], <8 x i16> [[TMP9]], <8 x i16> [[TMP11]])
; CHECK-NEXT: store <32 x i16> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
More information about the llvm-commits
mailing list