[llvm] 93ac788 - [VectorCombine] Fold deinterleave/interleave pairs (#211022)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 22:04:36 PDT 2026
Author: Jacob Crawley
Date: 2026-08-20T05:04:30Z
New Revision: 93ac788df8ffd9c9b8cf66c21249c1e6f96271e7
URL: https://github.com/llvm/llvm-project/commit/93ac788df8ffd9c9b8cf66c21249c1e6f96271e7
DIFF: https://github.com/llvm/llvm-project/commit/93ac788df8ffd9c9b8cf66c21249c1e6f96271e7.diff
LOG: [VectorCombine] Fold deinterleave/interleave pairs (#211022)
Fold matched `vector.deinterleaveN` / `vector.interleaveN` pairs through
equivalent elementwise operation chains.
When every deinterleaved field is transformed by the same elementwise
chain, rebuild that chain at the original vector width and remove the
deinterleave/interleave pair.
This eliminates redundant shuffle-like operations and enables improved
SVE code generation on AArch64 targets.
---------
Co-authored-by: Benjamin Maxwell <benjamin.maxwell at arm.com>
Added:
llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll
Modified:
llvm/lib/Transforms/Vectorize/VectorCombine.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index f6e58817cfb5a..dfe3070cfe87c 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -17,6 +17,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
@@ -162,6 +163,7 @@ class VectorCombine {
bool shrinkType(Instruction &I);
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
+ bool foldDeinterleaveInterleavePair(Instruction &I);
void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
@@ -5914,6 +5916,235 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
return true;
}
+/// Fold away a matched pair of vector.deinterleave/interleave intrinsics
+/// with a chain of elementwise operations on each between the
+/// deinterleave and interleave.
+///
+/// For example:
+/// ```
+/// %d = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %v)
+/// %f0 = extractvalue { <2 x i16>, <2 x i16> } %d, 0
+/// %f1 = extractvalue { <2 x i16>, <2 x i16> } %d, 1
+///
+/// %u0 = add <2 x i16> %f0, splat (i16 3)
+/// %u1 = add <2 x i16> %f1, splat (i16 3)
+///
+/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %u0, <2 x i16> %u1)
+/// ```
+/// Folds to:
+/// ```
+/// %r = add <4 x i16> %v, splat (i16 3)
+/// ```
+bool VectorCombine::foldDeinterleaveInterleavePair(Instruction &I) {
+ auto *Deinterleave = dyn_cast<IntrinsicInst>(&I);
+ if (!Deinterleave)
+ return false;
+
+ unsigned Factor =
+ getDeinterleaveIntrinsicFactor(Deinterleave->getIntrinsicID());
+ if (!Factor || Deinterleave->hasOperandBundles() ||
+ !Deinterleave->hasNUndroppableUses(Factor))
+ return false;
+
+ const Intrinsic::ID ExpectedInterleaveIID =
+ Intrinsic::getInterleaveIntrinsicID(Factor);
+
+ // Collect one extract for each deinterleaved field.
+ SmallVector<Use *, 8> CurrentUses(Factor, nullptr);
+ for (Use &U : Deinterleave->uses()) {
+ if (U.getUser()->isDroppable())
+ continue;
+
+ auto *Extract = dyn_cast<ExtractValueInst>(U.getUser());
+ if (!Extract || Extract->getNumIndices() != 1)
+ return false;
+
+ unsigned Index = *Extract->idx_begin();
+ if (Index >= Factor || CurrentUses[Index])
+ return false;
+
+ CurrentUses[Index] = &U;
+ }
+
+ using ElementwiseStep = SmallVector<Use *, 8>;
+ SmallVector<ElementwiseStep, 4> Steps;
+ IntrinsicInst *Interleave = nullptr;
+ unsigned NumVisited = 0;
+
+ auto GetNumDataOperands = [](Instruction *Inst) {
+ if (auto *CB = dyn_cast<CallBase>(Inst))
+ return CB->arg_size(); // Exclude callee operand and bundles.
+ return Inst->getNumOperands();
+ };
+
+ auto IsSupportedElementwise = [&](Instruction *Inst) {
+ auto *ResultTy = dyn_cast<VectorType>(Inst->getType());
+ if (!ResultTy || !isSafeToSpeculativelyExecute(Inst))
+ return false;
+
+ if (auto *II = dyn_cast<IntrinsicInst>(Inst)) {
+ if (II->hasOperandBundles() ||
+ !isTriviallyVectorizable(II->getIntrinsicID()))
+ return false;
+ } else if (!isa<BinaryOperator, UnaryOperator, CastInst, CmpInst,
+ SelectInst, FreezeInst>(Inst)) {
+ return false;
+ }
+
+ // Reject operations that change the element-count.
+ // E.g., bitcast <vscale x 4 x i16> %v to <vscale x 8 x i8>
+ for (unsigned Op = 0, E = GetNumDataOperands(Inst); Op != E; ++Op) {
+ auto *OperandTy = dyn_cast<VectorType>(Inst->getOperand(Op)->getType());
+ if (OperandTy &&
+ OperandTy->getElementCount() != ResultTy->getElementCount())
+ return false;
+ }
+
+ return true;
+ };
+
+ // Traverse the Factor use chains with a breadth-first search.
+ // At each level, expect every chain to perform the same operation with the
+ // preceding chain value at the same operand position, until they all reach
+ // the matching interleave.
+ while (NumVisited + Factor <= MaxInstrsToScan) {
+ NumVisited += Factor;
+
+ for (Use *&CurrentUse : CurrentUses) {
+ Use *NextUse = CurrentUse->getUser()->getSingleUndroppableUse();
+ auto *Next =
+ NextUse ? dyn_cast<Instruction>(NextUse->getUser()) : nullptr;
+ if (!Next)
+ return false;
+
+ CurrentUse = NextUse;
+ }
+
+ // Check whether every chain has reached the same interleave.
+ if (auto *II = dyn_cast<IntrinsicInst>(CurrentUses.front()->getUser());
+ II && II->getIntrinsicID() == ExpectedInterleaveIID) {
+ if (II->hasOperandBundles())
+ return false;
+
+ for (unsigned Index = 0; Index != Factor; ++Index)
+ if (CurrentUses[Index]->getUser() != II ||
+ CurrentUses[Index]->getOperandNo() != Index)
+ return false;
+
+ Interleave = II;
+ break;
+ }
+
+ auto *FirstInst = cast<Instruction>(CurrentUses.front()->getUser());
+ if (!IsSupportedElementwise(FirstInst))
+ return false;
+
+ unsigned ChainOperand = CurrentUses.front()->getOperandNo();
+ if (any_of(CurrentUses, [&](Use *U) {
+ auto *Inst = cast<Instruction>(U->getUser());
+ return Inst != FirstInst && (U->getOperandNo() != ChainOperand ||
+ !FirstInst->isSameOperationAs(Inst));
+ }))
+ return false;
+
+ auto GetSplatOrScalar = [](Value *V) {
+ return isa<VectorType>(V->getType()) ? getSplatValue(V) : V;
+ };
+
+ // Non-chain operands must be either the same scalar or splats of that
+ // scalar. This intentionally rejects
diff ering poison/undef or non-splat
+ // vector operands between chains.
+ for (unsigned Op = 0, E = GetNumDataOperands(FirstInst); Op != E; ++Op) {
+ if (Op == ChainOperand)
+ continue;
+
+ Value *CommonValue = GetSplatOrScalar(FirstInst->getOperand(Op));
+ if (!CommonValue || any_of(CurrentUses, [&](Use *U) {
+ Instruction *Inst = cast<Instruction>(U->getUser());
+ return Inst != FirstInst &&
+ GetSplatOrScalar(Inst->getOperand(Op)) != CommonValue;
+ }))
+ return false;
+ }
+
+ Steps.push_back(CurrentUses);
+ }
+
+ if (!Interleave)
+ return false;
+
+ // Rebuild the matched elementwise chain at the original vector width.
+ Value *WideValue = Deinterleave->getArgOperand(0);
+ ElementCount WideEC =
+ cast<VectorType>(WideValue->getType())->getElementCount();
+
+ auto CreateWideInstruction = [&](Instruction *NarrowInst,
+ ArrayRef<Value *> NewOperands,
+ VectorType *WideResultTy) -> Value * {
+ assert(IsSupportedElementwise(NarrowInst) &&
+ "Expected supported elementwise");
+ if (isa<BinaryOperator, UnaryOperator>(NarrowInst))
+ return Builder.CreateNAryOp(NarrowInst->getOpcode(), NewOperands);
+ if (auto *Cast = dyn_cast<CastInst>(NarrowInst))
+ return Builder.CreateCast(Cast->getOpcode(), NewOperands[0],
+ WideResultTy);
+ if (auto *Cmp = dyn_cast<CmpInst>(NarrowInst))
+ return Builder.CreateCmp(Cmp->getPredicate(), NewOperands[0],
+ NewOperands[1]);
+ if (isa<SelectInst>(NarrowInst))
+ return Builder.CreateSelect(NewOperands[0], NewOperands[1],
+ NewOperands[2]);
+ if (isa<FreezeInst>(NarrowInst))
+ return Builder.CreateFreeze(NewOperands[0]);
+ if (auto *II = dyn_cast<IntrinsicInst>(NarrowInst))
+ return Builder.CreateIntrinsic(WideResultTy, II->getIntrinsicID(),
+ NewOperands);
+ llvm_unreachable("Unsupported instruction");
+ };
+
+ // The BFS has succeeded and collected multiple levels of instructions that
+ // can be SLP-widened into a chain of wider instructions.
+ for (const ElementwiseStep &Step : Steps) {
+ Instruction *NarrowInst = cast<Instruction>(Step.front()->getUser());
+ unsigned ChainOperand = Step.front()->getOperandNo();
+
+ Builder.SetInsertPoint(NarrowInst);
+ Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
+
+ unsigned NumOperands = GetNumDataOperands(NarrowInst);
+ SmallVector<Value *, 4> NewOperands;
+ NewOperands.reserve(NumOperands);
+
+ for (unsigned Op = 0; Op != NumOperands; ++Op) {
+ Value *Operand = NarrowInst->getOperand(Op);
+
+ if (Op == ChainOperand)
+ Operand = WideValue;
+ else if (isa<VectorType>(Operand->getType()))
+ Operand = Builder.CreateVectorSplat(WideEC, getSplatValue(Operand));
+ NewOperands.push_back(Operand);
+ }
+
+ auto *WideResultTy =
+ VectorType::get(NarrowInst->getType()->getScalarType(), WideEC);
+ Value *NewValue =
+ CreateWideInstruction(NarrowInst, NewOperands, WideResultTy);
+
+ SmallVector<Value *> NarrowInsts =
+ map_to_vector(Step, [](Use *U) { return cast<Value>(U->getUser()); });
+ propagateIRFlags(NewValue, NarrowInsts);
+
+ if (auto *NewInst = dyn_cast<Instruction>(NewValue))
+ propagateMetadata(NewInst, NarrowInsts);
+
+ WideValue = NewValue;
+ }
+
+ assert(WideValue->getType() == Interleave->getType());
+ replaceValue(*Interleave, *WideValue);
+ return true;
+}
+
/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
@@ -5984,6 +6215,9 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
/// ```
bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
+ if (foldDeinterleaveInterleavePair(I))
+ return true;
+
// This pattern involves bitcast that is not compatible with big endian.
if (DL->isBigEndian())
return false;
diff --git a/llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll b/llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll
new file mode 100644
index 0000000000000..1ffe6bac7308f
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll
@@ -0,0 +1,717 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine %s -S -o - | FileCheck %s
+
+define <vscale x 12 x i16> @deinterleave3_interleave3_direct(<vscale x 12 x i16> %v) {
+; SCAN3-LABEL: define <vscale x 12 x i16> @deinterleave3_interleave3_direct(
+; SCAN3-SAME: <vscale x 12 x i16> [[V:%.*]]) {
+; SCAN3-NEXT: ret <vscale x 12 x i16> [[V]]
+; CHECK-LABEL: define <vscale x 12 x i16> @deinterleave3_interleave3_direct(
+; CHECK-SAME: <vscale x 12 x i16> [[V:%.*]]) {
+; CHECK-NEXT: ret <vscale x 12 x i16> [[V]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave3.nxv12i16(<vscale x 12 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %r = call <vscale x 12 x i16> @llvm.vector.interleave3.nxv12i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1, <vscale x 4 x i16> %f2)
+ ret <vscale x 12 x i16> %r
+}
+
+define <vscale x 16 x i8> @deinterleave4_lshr_trunc_interleave4(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @deinterleave4_lshr_trunc_interleave4(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <vscale x 16 x i16> [[V]], splat (i16 8)
+; CHECK-NEXT: [[R:%.*]] = trunc nuw <vscale x 16 x i16> [[TMP1]] to <vscale x 16 x i8>
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %f0, splat (i16 8)
+ %t0 = trunc nuw <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %s1 = lshr <vscale x 4 x i16> %f1, splat (i16 8)
+ %t1 = trunc nuw <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %s2 = lshr <vscale x 4 x i16> %f2, splat (i16 8)
+ %t2 = trunc nuw <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %s3 = lshr <vscale x 4 x i16> %f3, splat (i16 8)
+ %t3 = trunc nuw <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 16 x i16> @deinterleave4_five_step_chain_interleave4(<vscale x 16 x i32> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @deinterleave4_five_step_chain_interleave4(
+; CHECK-SAME: <vscale x 16 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = and <vscale x 16 x i32> [[V]], splat (i32 65535)
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <vscale x 16 x i32> [[TMP1]], splat (i32 3)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <vscale x 16 x i32> [[TMP2]], splat (i32 90)
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 16 x i32> [[TMP3]], splat (i32 7)
+; CHECK-NEXT: [[R:%.*]] = trunc <vscale x 16 x i32> [[TMP4]] to <vscale x 16 x i16>
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %v)
+ %f0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 3
+ %a0 = and <vscale x 4 x i32> %f0, splat (i32 65535)
+ %a1 = and <vscale x 4 x i32> %f1, splat (i32 65535)
+ %a2 = and <vscale x 4 x i32> %f2, splat (i32 65535)
+ %a3 = and <vscale x 4 x i32> %f3, splat (i32 65535)
+ %b0 = lshr <vscale x 4 x i32> %a0, splat (i32 3)
+ %b1 = lshr <vscale x 4 x i32> %a1, splat (i32 3)
+ %b2 = lshr <vscale x 4 x i32> %a2, splat (i32 3)
+ %b3 = lshr <vscale x 4 x i32> %a3, splat (i32 3)
+ %c0 = xor <vscale x 4 x i32> %b0, splat (i32 90)
+ %c1 = xor <vscale x 4 x i32> %b1, splat (i32 90)
+ %c2 = xor <vscale x 4 x i32> %b2, splat (i32 90)
+ %c3 = xor <vscale x 4 x i32> %b3, splat (i32 90)
+ %d0 = add <vscale x 4 x i32> %c0, splat (i32 7)
+ %d1 = add <vscale x 4 x i32> %c1, splat (i32 7)
+ %d2 = add <vscale x 4 x i32> %c2, splat (i32 7)
+ %d3 = add <vscale x 4 x i32> %c3, splat (i32 7)
+ %e0 = trunc <vscale x 4 x i32> %d0 to <vscale x 4 x i16>
+ %e1 = trunc <vscale x 4 x i32> %d1 to <vscale x 4 x i16>
+ %e2 = trunc <vscale x 4 x i32> %d2 to <vscale x 4 x i16>
+ %e3 = trunc <vscale x 4 x i32> %d3 to <vscale x 4 x i16>
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %e0, <vscale x 4 x i16> %e1, <vscale x 4 x i16> %e2, <vscale x 4 x i16> %e3)
+ ret <vscale x 16 x i16> %r
+}
+
+define <vscale x 14 x i16> @deinterleave7_sub_interleave7_lhs_splat(<vscale x 14 x i16> %v) {
+; CHECK-LABEL: define <vscale x 14 x i16> @deinterleave7_sub_interleave7_lhs_splat(
+; CHECK-SAME: <vscale x 14 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = sub <vscale x 14 x i16> splat (i16 1023), [[V]]
+; CHECK-NEXT: ret <vscale x 14 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave7.nxv14i16(<vscale x 14 x i16> %v)
+ %f0 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %f4 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 4
+ %f5 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 5
+ %f6 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 6
+ %u0 = sub <vscale x 2 x i16> splat (i16 1023), %f0
+ %u1 = sub <vscale x 2 x i16> splat (i16 1023), %f1
+ %u2 = sub <vscale x 2 x i16> splat (i16 1023), %f2
+ %u3 = sub <vscale x 2 x i16> splat (i16 1023), %f3
+ %u4 = sub <vscale x 2 x i16> splat (i16 1023), %f4
+ %u5 = sub <vscale x 2 x i16> splat (i16 1023), %f5
+ %u6 = sub <vscale x 2 x i16> splat (i16 1023), %f6
+ %r = call <vscale x 14 x i16> @llvm.vector.interleave7.nxv14i16(<vscale x 2 x i16> %u0, <vscale x 2 x i16> %u1, <vscale x 2 x i16> %u2, <vscale x 2 x i16> %u3, <vscale x 2 x i16> %u4, <vscale x 2 x i16> %u5, <vscale x 2 x i16> %u6)
+ ret <vscale x 14 x i16> %r
+}
+
+define <vscale x 16 x i32> @deinterleave8_and_interleave8(<vscale x 16 x i32> %v) {
+; CHECK-LABEL: define <vscale x 16 x i32> @deinterleave8_and_interleave8(
+; CHECK-SAME: <vscale x 16 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = and <vscale x 16 x i32> [[V]], splat (i32 255)
+; CHECK-NEXT: ret <vscale x 16 x i32> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave8.nxv16i32(<vscale x 16 x i32> %v)
+ %f0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %f2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+ %f3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
+ %f4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 4
+ %f5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 5
+ %f6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 6
+ %f7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 7
+ %u0 = and <vscale x 2 x i32> %f0, splat (i32 255)
+ %u1 = and <vscale x 2 x i32> %f1, splat (i32 255)
+ %u2 = and <vscale x 2 x i32> %f2, splat (i32 255)
+ %u3 = and <vscale x 2 x i32> %f3, splat (i32 255)
+ %u4 = and <vscale x 2 x i32> %f4, splat (i32 255)
+ %u5 = and <vscale x 2 x i32> %f5, splat (i32 255)
+ %u6 = and <vscale x 2 x i32> %f6, splat (i32 255)
+ %u7 = and <vscale x 2 x i32> %f7, splat (i32 255)
+ %r = call <vscale x 16 x i32> @llvm.vector.interleave8.nxv16i32(<vscale x 2 x i32> %u0, <vscale x 2 x i32> %u1, <vscale x 2 x i32> %u2, <vscale x 2 x i32> %u3, <vscale x 2 x i32> %u4, <vscale x 2 x i32> %u5, <vscale x 2 x i32> %u6, <vscale x 2 x i32> %u7)
+ ret <vscale x 16 x i32> %r
+}
+
+define <vscale x 16 x i16> @deinterleave4_zext_interleave4(<vscale x 16 x i8> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @deinterleave4_zext_interleave4(
+; CHECK-SAME: <vscale x 16 x i8> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = zext <vscale x 16 x i8> [[V]] to <vscale x 16 x i16>
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave4.nxv16i8(<vscale x 16 x i8> %v)
+ %f0 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 3
+ %u0 = zext <vscale x 4 x i8> %f0 to <vscale x 4 x i16>
+ %u1 = zext <vscale x 4 x i8> %f1 to <vscale x 4 x i16>
+ %u2 = zext <vscale x 4 x i8> %f2 to <vscale x 4 x i16>
+ %u3 = zext <vscale x 4 x i8> %f3 to <vscale x 4 x i16>
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1, <vscale x 4 x i16> %u2, <vscale x 4 x i16> %u3)
+ ret <vscale x 16 x i16> %r
+}
+
+define <16 x i8> @fixed_deinterleave4_lshr_trunc_interleave4(<16 x i16> %v) {
+; CHECK-LABEL: define <16 x i8> @fixed_deinterleave4_lshr_trunc_interleave4(
+; CHECK-SAME: <16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V]], splat (i16 8)
+; CHECK-NEXT: [[R:%.*]] = trunc <16 x i16> [[TMP1]] to <16 x i8>
+; CHECK-NEXT: ret <16 x i8> [[R]]
+;
+ %d = call { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.vector.deinterleave4.v16i16(<16 x i16> %v)
+ %f0 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 0
+ %f1 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 1
+ %f2 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 2
+ %f3 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 3
+ %s0 = lshr <4 x i16> %f0, splat (i16 8)
+ %t0 = trunc <4 x i16> %s0 to <4 x i8>
+ %s1 = lshr <4 x i16> %f1, splat (i16 8)
+ %t1 = trunc <4 x i16> %s1 to <4 x i8>
+ %s2 = lshr <4 x i16> %f2, splat (i16 8)
+ %t2 = trunc <4 x i16> %s2 to <4 x i8>
+ %s3 = lshr <4 x i16> %f3, splat (i16 8)
+ %t3 = trunc <4 x i16> %s3 to <4 x i8>
+ %r = call <16 x i8> @llvm.vector.interleave4.v16i8(<4 x i8> %t0, <4 x i8> %t1, <4 x i8> %t2, <4 x i8> %t3)
+ ret <16 x i8> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_add_separate_splats_interleave2(<vscale x 8 x i16> %v, i16 %x) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_add_separate_splats_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], i16 [[X:%.*]]) {
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[X]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[DOTSPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <vscale x 8 x i16> [[V]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %insert0 = insertelement <vscale x 4 x i16> poison, i16 %x, i64 0
+ %splat0 = shufflevector <vscale x 4 x i16> %insert0, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ %insert1 = insertelement <vscale x 4 x i16> poison, i16 %x, i64 0
+ %splat1 = shufflevector <vscale x 4 x i16> %insert1, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ %u0 = add <vscale x 4 x i16> %f0, %splat0
+ %u1 = add <vscale x 4 x i16> %f1, %splat1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x double> @deinterleave2_fpext_interleave2(<vscale x 8 x float> %v) {
+; CHECK-LABEL: define <vscale x 8 x double> @deinterleave2_fpext_interleave2(
+; CHECK-SAME: <vscale x 8 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = fpext <vscale x 8 x float> [[V]] to <vscale x 8 x double>
+; CHECK-NEXT: ret <vscale x 8 x double> [[R]]
+;
+ %d = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %v)
+ %f0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 1
+ %u0 = fpext <vscale x 4 x float> %f0 to <vscale x 4 x double>
+ %u1 = fpext <vscale x 4 x float> %f1 to <vscale x 4 x double>
+ %r = call <vscale x 8 x double> @llvm.vector.interleave2.nxv8f64(<vscale x 4 x double> %u0, <vscale x 4 x double> %u1)
+ ret <vscale x 8 x double> %r
+}
+
+define <vscale x 8 x float> @deinterleave2_fneg_interleave2(<vscale x 8 x float> %v) {
+; CHECK-LABEL: define <vscale x 8 x float> @deinterleave2_fneg_interleave2(
+; CHECK-SAME: <vscale x 8 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = fneg <vscale x 8 x float> [[V]]
+; CHECK-NEXT: ret <vscale x 8 x float> [[R]]
+;
+ %d = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %v)
+ %f0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 1
+ %u0 = fneg <vscale x 4 x float> %f0
+ %u1 = fneg <vscale x 4 x float> %f1
+ %r = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %u0, <vscale x 4 x float> %u1)
+ ret <vscale x 8 x float> %r
+}
+
+define <vscale x 8 x float> @deinterleave2_bitcast_interleave2(<vscale x 8 x i32> %v) {
+; CHECK-LABEL: define <vscale x 8 x float> @deinterleave2_bitcast_interleave2(
+; CHECK-SAME: <vscale x 8 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 8 x i32> [[V]] to <vscale x 8 x float>
+; CHECK-NEXT: ret <vscale x 8 x float> [[R]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %v)
+ %f0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %u0 = bitcast <vscale x 4 x i32> %f0 to <vscale x 4 x float>
+ %u1 = bitcast <vscale x 4 x i32> %f1 to <vscale x 4 x float>
+ %r = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %u0, <vscale x 4 x float> %u1)
+ ret <vscale x 8 x float> %r
+}
+
+define <vscale x 8 x i1> @deinterleave2_icmp_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i1> @deinterleave2_icmp_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = icmp eq <vscale x 8 x i16> [[V]], zeroinitializer
+; CHECK-NEXT: ret <vscale x 8 x i1> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = icmp eq <vscale x 4 x i16> %f0, zeroinitializer
+ %u1 = icmp eq <vscale x 4 x i16> %f1, zeroinitializer
+ %r = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %u0, <vscale x 4 x i1> %u1)
+ ret <vscale x 8 x i1> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_select_interleave2(<vscale x 8 x i1> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_select_interleave2(
+; CHECK-SAME: <vscale x 8 x i1> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = select <vscale x 8 x i1> [[V]], <vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> splat (i16 1)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %v)
+ %f0 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %d, 1
+ %u0 = select <vscale x 4 x i1> %f0, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> splat (i16 1)
+ %u1 = select <vscale x 4 x i1> %f1, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> splat (i16 1)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_freeze_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_freeze_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = freeze <vscale x 8 x i16> [[V]]
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = freeze <vscale x 4 x i16> %f0
+ %u1 = freeze <vscale x 4 x i16> %f1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_abs_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_abs_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> [[V]], i1 false)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f0, i1 false)
+ %u1 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f1, i1 false)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_intersect_flags_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_intersect_flags_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = add <vscale x 8 x i16> [[V]], splat (i16 1)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add nuw <vscale x 4 x i16> %f0, splat (i16 1)
+ %u1 = add <vscale x 4 x i16> %f1, splat (i16 1)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; TODO: Handle non-speculatable instructions.
+; We can only support these when all non-speculatable instructions are in the same block.
+define <vscale x 16 x i16> @control_flow_sdiv(<vscale x 16 x i16> %v, i1 %cond, i16 %splat_value) {
+; CHECK-LABEL: define <vscale x 16 x i16> @control_flow_sdiv(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]], i1 [[COND:%.*]], i16 [[SPLAT_VALUE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[INSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[SPLAT_VALUE]], i64 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[INSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[A0:%.*]] = sdiv <vscale x 4 x i16> [[F0]], [[SPLAT]]
+; CHECK-NEXT: [[A1:%.*]] = sdiv <vscale x 4 x i16> [[F1]], [[SPLAT]]
+; CHECK-NEXT: [[A2:%.*]] = sdiv <vscale x 4 x i16> [[F2]], [[SPLAT]]
+; CHECK-NEXT: [[A3:%.*]] = sdiv <vscale x 4 x i16> [[F3]], [[SPLAT]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[A0]], <vscale x 4 x i16> [[A1]], <vscale x 4 x i16> [[A2]], <vscale x 4 x i16> [[A3]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: ret <vscale x 16 x i16> zeroinitializer
+;
+entry:
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %insert = insertelement <vscale x 4 x i16> poison, i16 %splat_value, i64 0
+ %splat = shufflevector <vscale x 4 x i16> %insert, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ br i1 %cond, label %then, label %else
+
+then:
+ %a0 = sdiv <vscale x 4 x i16> %f0, %splat
+ %a1 = sdiv <vscale x 4 x i16> %f1, %splat
+ %a2 = sdiv <vscale x 4 x i16> %f2, %splat
+ %a3 = sdiv <vscale x 4 x i16> %f3, %splat
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %a0, <vscale x 4 x i16> %a1, <vscale x 4 x i16> %a2, <vscale x 4 x i16> %a3)
+ ret <vscale x 16 x i16> %r
+
+else:
+ ret <vscale x 16 x i16> zeroinitializer
+}
+
+define <vscale x 8 x float> @fpmath_metadata(<vscale x 8 x float> %v) {
+; CHECK-LABEL: define <vscale x 8 x float> @fpmath_metadata(
+; CHECK-SAME: <vscale x 8 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = fadd <vscale x 8 x float> [[V]], splat (float 1.000000e+00), !fpmath [[META0:![0-9]+]]
+; CHECK-NEXT: ret <vscale x 8 x float> [[R]]
+;
+ %d = call { <vscale x 4 x float>, <vscale x 4 x float> }
+ @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %v)
+ %f0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 1
+ %u0 = fadd <vscale x 4 x float> %f0, splat (float 1.0), !fpmath !{float 2.5}
+ %u1 = fadd <vscale x 4 x float> %f1, splat (float 1.0), !fpmath !{float 2.5}
+ %r = call <vscale x 8 x float>
+ @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %u0, <vscale x 4 x float> %u1)
+ ret <vscale x 8 x float> %r
+}
+
+define <8 x i1> @icmp_i1_fields(<8 x i1> %v) {
+; CHECK-LABEL: define <8 x i1> @icmp_i1_fields(
+; CHECK-SAME: <8 x i1> [[V:%.*]]) {
+; CHECK-NEXT: ret <8 x i1> [[V]]
+;
+ %d = call { <4 x i1>, <4 x i1> } @llvm.vector.deinterleave2.v8i1(<8 x i1> %v)
+ %f0 = extractvalue { <4 x i1>, <4 x i1> } %d, 0
+ %f1 = extractvalue { <4 x i1>, <4 x i1> } %d, 1
+ %u0 = icmp eq <4 x i1> %f0, splat (i1 true)
+ %u1 = icmp eq <4 x i1> %f1, splat (i1 true)
+ %r = call <8 x i1> @llvm.vector.interleave2.v8i1(<4 x i1> %u0, <4 x i1> %u1)
+ ret <8 x i1> %r
+}
+
+define <4 x i32> @deinterleave2_select_scalar_condition_interleave2(
+; CHECK-LABEL: define <4 x i32> @deinterleave2_select_scalar_condition_interleave2(
+; CHECK-SAME: <4 x i32> [[V:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[COND]], <4 x i32> [[V]], <4 x i32> splat (i32 7)
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ <4 x i32> %v, i32 %x, i32 %y) {
+ %cond = icmp eq i32 %x, %y
+ %d = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %v)
+ %f0 = extractvalue { <2 x i32>, <2 x i32> } %d, 0
+ %f1 = extractvalue { <2 x i32>, <2 x i32> } %d, 1
+ %u0 = select i1 %cond, <2 x i32> %f0, <2 x i32> splat (i32 7)
+ %u1 = select i1 %cond, <2 x i32> %f1, <2 x i32> splat (i32 7)
+ %r = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %u0, <2 x i32> %u1)
+ ret <4 x i32> %r
+}
+
+; Negative test: operand bundles on the deinterleave must be preserved.
+define <vscale x 8 x i16> @negative_deinterleave2_operand_bundle(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_operand_bundle(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[F0]], <vscale x 4 x i16> [[F1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v) [ "deopt"(i32 0) ]
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: one chain uses a
diff erent shift amount, so the fold must not happen.
+define <vscale x 16 x i8> @negative_deinterleave4_mismatched_shift_amount(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @negative_deinterleave4_mismatched_shift_amount(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 4 x i16> [[F0]], splat (i16 8)
+; CHECK-NEXT: [[T0:%.*]] = trunc nuw <vscale x 4 x i16> [[S0]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 4 x i16> [[F1]], splat (i16 8)
+; CHECK-NEXT: [[T1:%.*]] = trunc nuw <vscale x 4 x i16> [[S1]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S2:%.*]] = lshr <vscale x 4 x i16> [[F2]], splat (i16 7)
+; CHECK-NEXT: [[T2:%.*]] = trunc nuw <vscale x 4 x i16> [[S2]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S3:%.*]] = lshr <vscale x 4 x i16> [[F3]], splat (i16 8)
+; CHECK-NEXT: [[T3:%.*]] = trunc nuw <vscale x 4 x i16> [[S3]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> [[T0]], <vscale x 4 x i8> [[T1]], <vscale x 4 x i8> [[T2]], <vscale x 4 x i8> [[T3]])
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %f0, splat (i16 8)
+ %t0 = trunc nuw <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %s1 = lshr <vscale x 4 x i16> %f1, splat (i16 8)
+ %t1 = trunc nuw <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %s2 = lshr <vscale x 4 x i16> %f2, splat (i16 7)
+ %t2 = trunc nuw <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %s3 = lshr <vscale x 4 x i16> %f3, splat (i16 8)
+ %t3 = trunc nuw <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ ret <vscale x 16 x i8> %r
+}
+
+; Negative test - the fold shouldn't be generated as the deitnerleaved fields are passed in the wrong order.
+define <vscale x 4 x i16> @negative_deinterleave2_reordered_fields(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @negative_deinterleave2_reordered_fields(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t1, <vscale x 2 x i16> %t0)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test: reusing one extracted field and skipping another means the fold must not happen.
+define <vscale x 16 x i16> @negative_deinterleave4_duplicate_extract_operand(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @negative_deinterleave4_duplicate_extract_operand(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F0_DUP:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[F0]], <vscale x 4 x i16> [[F1]], <vscale x 4 x i16> [[F2]], <vscale x 4 x i16> [[F0_DUP]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f0.dup = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1, <vscale x 4 x i16> %f2, <vscale x 4 x i16> %f0.dup)
+ ret <vscale x 16 x i16> %r
+}
+
+; Negative test: changing the extracted-value operand position in one chain means the fold must not happen.
+define <vscale x 16 x i16> @negative_deinterleave4_mismatched_operand_position(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @negative_deinterleave4_mismatched_operand_position(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[U0:%.*]] = sub <vscale x 4 x i16> [[F0]], splat (i16 5)
+; CHECK-NEXT: [[U1:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F1]]
+; CHECK-NEXT: [[U2:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F2]]
+; CHECK-NEXT: [[U3:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F3]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]], <vscale x 4 x i16> [[U2]], <vscale x 4 x i16> [[U3]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %u0 = sub <vscale x 4 x i16> %f0, splat (i16 5)
+ %u1 = sub <vscale x 4 x i16> splat (i16 5), %f1
+ %u2 = sub <vscale x 4 x i16> splat (i16 5), %f2
+ %u3 = sub <vscale x 4 x i16> splat (i16 5), %f3
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1, <vscale x 4 x i16> %u2, <vscale x 4 x i16> %u3)
+ ret <vscale x 16 x i16> %r
+}
+
+; Negative test: compare predicates are part of the operation and must match.
+define <vscale x 8 x i1> @negative_deinterleave2_mismatched_predicates(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i1> @negative_deinterleave2_mismatched_predicates(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = icmp eq <vscale x 4 x i16> [[F0]], zeroinitializer
+; CHECK-NEXT: [[U1:%.*]] = icmp ne <vscale x 4 x i16> [[F1]], zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[U0]], <vscale x 4 x i1> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i1> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = icmp eq <vscale x 4 x i16> %f0, zeroinitializer
+ %u1 = icmp ne <vscale x 4 x i16> %f1, zeroinitializer
+ %r = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %u0, <vscale x 4 x i1> %u1)
+ ret <vscale x 8 x i1> %r
+}
+
+; Negative test: non-data intrinsic operands must match across every chain.
+define <vscale x 8 x i16> @negative_deinterleave2_mismatched_intrinsic_scalar(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_mismatched_intrinsic_scalar(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> [[F0]], i1 false)
+; CHECK-NEXT: [[U1:%.*]] = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> [[F1]], i1 true)
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f0, i1 false)
+ %u1 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f1, i1 true)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: a bitcast that changes the element count is not elementwise.
+define <vscale x 16 x i8> @negative_deinterleave2_element_count_changing_bitcast(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @negative_deinterleave2_element_count_changing_bitcast(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = bitcast <vscale x 4 x i16> [[F0]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[U1:%.*]] = bitcast <vscale x 4 x i16> [[F1]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[U0]], <vscale x 8 x i8> [[U1]])
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = bitcast <vscale x 4 x i16> %f0 to <vscale x 8 x i8>
+ %u1 = bitcast <vscale x 4 x i16> %f1 to <vscale x 8 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %u0, <vscale x 8 x i8> %u1)
+ ret <vscale x 16 x i8> %r
+}
+
+; Negative test: arbitrary vector operands cannot be widened like splats.
+define <vscale x 8 x i16> @negative_deinterleave2_non_splat_vector_operand(<vscale x 8 x i16> %v, <vscale x 4 x i16> %x0, <vscale x 4 x i16> %x1) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_non_splat_vector_operand(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], <vscale x 4 x i16> [[X0:%.*]], <vscale x 4 x i16> [[X1:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = add <vscale x 4 x i16> [[F0]], [[X0]]
+; CHECK-NEXT: [[U1:%.*]] = add <vscale x 4 x i16> [[F1]], [[X1]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add <vscale x 4 x i16> %f0, %x0
+ %u1 = add <vscale x 4 x i16> %f1, %x1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: the fold must not remove an intermediate value with another use.
+define <vscale x 8 x i16> @negative_deinterleave2_extra_intermediate_use(<vscale x 8 x i16> %v, ptr %p) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_extra_intermediate_use(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = add <vscale x 4 x i16> [[F0]], splat (i16 1)
+; CHECK-NEXT: [[U1:%.*]] = add <vscale x 4 x i16> [[F1]], splat (i16 1)
+; CHECK-NEXT: store <vscale x 4 x i16> [[U0]], ptr [[P]], align 8
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add <vscale x 4 x i16> %f0, splat (i16 1)
+ %u1 = add <vscale x 4 x i16> %f1, splat (i16 1)
+ store <vscale x 4 x i16> %u0, ptr %p
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: merging the chains at an intermediate instruction prevents the elementwise-chain fold.
+define <vscale x 8 x i16> @negative_deinterleave2(
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U:%.*]] = add <vscale x 4 x i16> [[F0]], [[F1]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U]], <vscale x 4 x i16> [[U]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ <vscale x 8 x i16> %v) {
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u = add <vscale x 4 x i16> %f0, %f1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u, <vscale x 4 x i16> %u)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: Don't allow non-speculatable users in
diff erent blocks.
+define <vscale x 16 x i16> @non_speculatable_uses_mixed_blocks(<vscale x 16 x i16> %v, i1 %cond, i16 %splat_value) {
+; CHECK-LABEL: define <vscale x 16 x i16> @non_speculatable_uses_mixed_blocks(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]], i1 [[COND:%.*]], i16 [[SPLAT_VALUE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[INSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[SPLAT_VALUE]], i64 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[INSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[A0:%.*]] = sdiv <vscale x 4 x i16> [[SPLAT]], [[F0]]
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[A1:%.*]] = sdiv <vscale x 4 x i16> [[SPLAT]], [[F1]]
+; CHECK-NEXT: [[A2:%.*]] = sdiv <vscale x 4 x i16> [[SPLAT]], [[F2]]
+; CHECK-NEXT: [[A3:%.*]] = sdiv <vscale x 4 x i16> [[SPLAT]], [[F3]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[A0]], <vscale x 4 x i16> [[A1]], <vscale x 4 x i16> [[A2]], <vscale x 4 x i16> [[A3]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: ret <vscale x 16 x i16> zeroinitializer
+;
+entry:
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %insert = insertelement <vscale x 4 x i16> poison, i16 %splat_value, i64 0
+ %splat = shufflevector <vscale x 4 x i16> %insert, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ %a0 = sdiv <vscale x 4 x i16> %splat, %f0
+ br i1 %cond, label %then, label %else
+
+then:
+ %a1 = sdiv <vscale x 4 x i16> %splat, %f1
+ %a2 = sdiv <vscale x 4 x i16> %splat, %f2
+ %a3 = sdiv <vscale x 4 x i16> %splat, %f3
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %a0, <vscale x 4 x i16> %a1, <vscale x 4 x i16> %a2, <vscale x 4 x i16> %a3)
+ ret <vscale x 16 x i16> %r
+
+else:
+ ret <vscale x 16 x i16> zeroinitializer
+}
+
+;.
+; CHECK: [[META0]] = !{float 2.500000e+00}
+;.
More information about the llvm-commits
mailing list