[llvm] [IR] Add llvm.vector.shuffle intrinsic for vector shuffles with runtime masks Part 1. (PR #219259)
Oscar Smith via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 22:24:17 PDT 2026
https://github.com/oscardssmith updated https://github.com/llvm/llvm-project/pull/219259
>From 9f0685a55c7c13fb858dc44f8bc34cddc7267366 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscar.smith at juliahub.com>
Date: Wed, 23 Sep 2026 05:11:08 +0000
Subject: [PATCH] [IR][SelectionDAG] Add llvm.vector.shuffle intrinsic
Add an intrinsic that permutes a vector by a mask that need not be a
constant:
<N x ty> @llvm.vector.shuffle(<N x ty> %vec, <N x iK> %mask)
Result element i is %vec[zext(%mask[i])], or poison if that index is out
of range. Poison is the one out-of-range behavior that x86 VPERMV/PSHUFB,
AArch64 TBL and RISC-V vrgather can all implement without fixup code, and
it lets the intrinsic be speculatable. The mask element type is overloaded
so targets can take their native index width directly.
This follows the RFC at
https://discourse.llvm.org/t/rfc-ir-ability-to-shuffle-vectors-with-dynamic-mask/91282
and is the first part of #208537. It adds the intrinsic, its verifier
check, and the ISD::VECTOR_SHUFFLE_VAR node with type legalization and a
generic expansion through a stack temporary. No target lowers the node
natively yet, and scalable vectors are not supported until one does.
Target lowering, the cost model and combines will follow separately.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/docs/LangRef.md | 37 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 6 +
llvm/include/llvm/CodeGen/TargetLowering.h | 4 +
llvm/include/llvm/IR/Intrinsics.td | 5 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 21 +
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 6 +
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 107 ++
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 16 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 +
.../SelectionDAG/SelectionDAGDumper.cpp | 1 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 57 +
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 +-
llvm/lib/IR/Verifier.cpp | 8 +
.../AArch64/vector-shuffle-var-expand.ll | 489 +++++++
.../CodeGen/X86/vector-shuffle-var-expand.ll | 129 ++
.../X86/vector-shuffle-var-legalize.ll | 1155 +++++++++++++++++
llvm/test/Verifier/vector-shuffle-var.ll | 19 +
18 files changed, 2072 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
create mode 100644 llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
create mode 100644 llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
create mode 100644 llvm/test/Verifier/vector-shuffle-var.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index a4816d337e02b3..f3bed2d190f2c3 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -21489,6 +21489,43 @@ VecT compress(VecT vec, VecT mask, VecT passthru) {
}
```
+(int_vector_shuffle)=
+
+#### '`llvm.vector.shuffle.*`' Intrinsic
+
+##### Syntax:
+
+This is an overloaded intrinsic.
+
+```llvm
+declare <8 x i32> @llvm.vector.shuffle.v8i32.v8i8(<8 x i32> %vec, <8 x i8> %mask)
+declare <vscale x 4 x float> @llvm.vector.shuffle.nxv4f32.nxv4i16(<vscale x 4 x float> %vec, <vscale x 4 x i16> %mask)
+```
+
+##### Overview:
+
+The '`llvm.vector.shuffle.*`' intrinsics permute the elements of a vector
+according to a mask that does not need to be a constant.
+
+For example:
+
+```text
+llvm.vector.shuffle(<A,B,C,D>, <3,0,0,7>) ==> <D,A,A,poison>
+```
+
+##### Arguments:
+
+The first operand is the vector to permute. The result has the same type. The
+second operand is the mask, a vector of integers with the same number of
+elements as the first operand.
+
+##### Semantics:
+
+Element `i` of the result is element `%mask[i]` of `%vec`, where `%mask[i]` is
+treated as an unsigned index. If `%mask[i]` is poison, or is greater than or
+equal to the number of elements in `%vec`, element `i` of the result is a
+{ref}`poison value <poisonvalues>`.
+
#### '`llvm.experimental.vector.match.*`' Intrinsic
##### Syntax:
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 1af501d4c8dd05..27ac79d59302e5 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -701,6 +701,12 @@ enum NodeType {
/// If passthru is undef, ?s remain undefined.
VECTOR_COMPRESS,
+ /// VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer
+ /// vector with the same number of elements as VEC. Result element i is
+ /// VEC[MASK[i]], with MASK[i] treated as unsigned, or poison if MASK[i] is
+ /// out of range.
+ VECTOR_SHUFFLE_VAR,
+
/// MULHU/MULHS - Multiply high - Multiply two integers of type iN,
/// producing an unsigned/signed value of type i[2*N], then return the top
/// part.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c08d0e53ec34b3..34aaf89788e8d7 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -6053,6 +6053,10 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// temporarily, advance store position, before re-loading the final vector.
SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const;
+ /// Expand a VECTOR_SHUFFLE_VAR by storing the source vector to the stack and
+ /// loading each result element from it.
+ SDValue expandVECTOR_SHUFFLE_VAR(SDNode *Node, SelectionDAG &DAG) const;
+
/// Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for
/// each active lane (i), getting the maximum and subtracting it from VL.
SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 26b7c772eb2446..5b15d1303d867e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2790,6 +2790,11 @@ def int_vector_splice_right
[LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
[IntrNoMem, IntrSpeculatable]>;
+def int_vector_shuffle:
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [LLVMMatchType<0>, llvm_any_vector_int_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
//===---------- Intrinsics to query properties of scalable vectors --------===//
def int_vscale : DefaultAttrsIntrinsic<[llvm_anyint_ty],
[],
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index a90bb06fb424dc..a3657697f10021 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -90,6 +90,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
Res = PromoteIntRes_VECTOR_COMPRESS(N);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = PromoteIntRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::SELECT:
case ISD::VSELECT:
case ISD::VP_MERGE:
@@ -1039,6 +1042,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(SDNode *N) {
N->getOperand(1), Passthru);
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ SDValue V = GetPromotedInteger(N->getOperand(0));
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), V.getValueType(), V,
+ N->getOperand(1));
+}
+
/// Promote the overflow flag of an overflowing arithmetic node.
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
// Change the return type of the boolean result while obeying
@@ -2050,6 +2059,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_COMPRESS:
Res = PromoteIntOp_VECTOR_COMPRESS(N, OpNo);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = PromoteIntOp_VECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
case ISD::BF16_TO_FP:
case ISD::FP16_TO_FP:
@@ -2633,6 +2645,15 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_COMPRESS(SDNode *N,
return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), VT, Vec, Mask, Passthru);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_SHUFFLE_VAR(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "The source shares the result's type, so promoting it "
+ "is a result promotion.");
+ // Mask elements are unsigned.
+ SDValue Mask = ZExtPromotedInteger(N->getOperand(1));
+ return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Mask), 0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index c9583d99ddbf43..18658dd0d12e37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -317,6 +317,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
SDValue PromoteIntRes_VECTOR_COMPRESS(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue PromoteIntRes_Overflow(SDNode *N);
SDValue PromoteIntRes_FFREXP(SDNode *N);
SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
@@ -402,6 +403,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
SDValue PromoteIntOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);
+ SDValue PromoteIntOp_VECTOR_SHUFFLE_VAR(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
SDValue PromoteIntOp_FIX(SDNode *N);
SDValue PromoteIntOp_ExpOp(SDNode *N);
@@ -837,6 +839,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_BITCAST(SDNode *N);
SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
+ SDValue ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
SDValue ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N);
SDValue ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N);
@@ -949,6 +952,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_Gather(MemSDNode *VPGT, SDValue &Lo, SDValue &Hi,
bool SplitSETCC = false);
void SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -976,6 +980,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_UnaryOp(SDNode *N);
SDValue SplitVecOp_TruncateHelper(SDNode *N);
SDValue SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);
+ SDValue LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_BITCAST(SDNode *N);
SDValue SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
@@ -1057,6 +1062,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N);
SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);
SDValue WidenVecRes_VECTOR_COMPRESS(SDNode *N);
+ SDValue WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
SDValue WidenVecRes_VP_GATHER(VPGatherSDNode* N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 5e3f252fdd3d48..26ecf97b9231e0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -479,6 +479,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::FP_TO_UINT_SAT:
case ISD::MGATHER:
case ISD::VECTOR_COMPRESS:
+ case ISD::VECTOR_SHUFFLE_VAR:
case ISD::SCMP:
case ISD::UCMP:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
@@ -1303,6 +1304,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECTOR_COMPRESS:
Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG));
return;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Results.push_back(TLI.expandVECTOR_SHUFFLE_VAR(Node, DAG));
+ return;
case ISD::CTTZ_ELTS:
case ISD::CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandCttzElts(Node, DAG));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 56ad7c65b48d1b..a0e2846c7f1bf6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -65,6 +65,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ R = ScalarizeVecRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
case ISD::CONVERT_FROM_ARBITRARY_FP:
R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
@@ -491,6 +494,11 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
N->getOperand(0), N->getOperand(1));
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ // Index 0 is the only in-range index.
+ return GetScalarizedVector(N->getOperand(0));
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
SDLoc DL(N);
SDValue Op = N->getOperand(0);
@@ -881,6 +889,10 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::FAKE_USE:
Res = ScalarizeVecOp_FAKE_USE(N);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ // Index 0 is the only in-range index.
+ Res = N->getOperand(0);
+ break;
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::SIGN_EXTEND:
@@ -1446,6 +1458,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ SplitVecRes_VECTOR_SHUFFLE_VAR(N, Lo, Hi);
+ break;
case ISD::SETCC:
SplitVecRes_SETCC(N, Lo, Hi);
break;
@@ -2928,6 +2943,58 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
std::tie(Lo, Hi) = DAG.SplitVector(Compressed, DL);
}
+void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ SDLoc DL(N);
+ SDValue Src = N->getOperand(0), Mask = N->getOperand(1);
+ EVT MaskVT = Mask.getValueType();
+
+ // TODO: Support scalable vectors, which needs a mask element type that can
+ // hold the half element count.
+ if (MaskVT.isScalableVector())
+ report_fatal_error("Cannot split scalable vector_shuffle_var.");
+
+ SDValue SrcLo, SrcHi;
+ GetSplitVector(Src, SrcLo, SrcHi);
+
+ SDValue MaskLo, MaskHi;
+ if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
+ GetSplitVector(Mask, MaskLo, MaskHi);
+ else
+ std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
+
+ EVT HalfVT = SrcLo.getValueType();
+ EVT HalfMaskVT = MaskLo.getValueType();
+ unsigned HalfElts = HalfVT.getVectorNumElements();
+
+ // If the mask can't index past the low half, the high half is unused.
+ if (APInt::getMaxValue(MaskVT.getScalarSizeInBits()).ult(HalfElts)) {
+ Lo = DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, MaskLo);
+ Hi = DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, MaskHi);
+ return;
+ }
+
+ // Otherwise shuffle each source half and select between them:
+ // Half[i] = Idx[i] <u HalfElts ? SrcLo[Idx[i]] : SrcHi[Idx[i] - HalfElts]
+ SDValue HalfEltsV = DAG.getConstant(HalfElts, DL, HalfMaskVT);
+ EVT CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ HalfMaskVT);
+ EVT CCVT =
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HalfVT);
+ auto ShuffleHalf = [&](SDValue Idx) {
+ SDValue FromLo =
+ DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, Idx);
+ SDValue HiIdx = DAG.getNode(ISD::SUB, DL, HalfMaskVT, Idx, HalfEltsV);
+ SDValue FromHi =
+ DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcHi, HiIdx);
+ SDValue InLo = DAG.getSetCC(DL, CmpVT, Idx, HalfEltsV, ISD::SETULT);
+ InLo = DAG.getBoolExtOrTrunc(InLo, DL, CCVT, HalfMaskVT);
+ return DAG.getSelect(DL, HalfVT, InLo, FromLo, FromHi);
+ };
+ Lo = ShuffleHalf(MaskLo);
+ Hi = ShuffleHalf(MaskHi);
+}
+
void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
assert(N->getValueType(0).isVector() &&
N->getOperand(0).getValueType().isVector() &&
@@ -3850,6 +3917,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_COMPRESS:
Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::STRICT_SINT_TO_FP:
case ISD::STRICT_UINT_TO_FP:
case ISD::SINT_TO_FP:
@@ -4073,6 +4143,26 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VecVT, Lo, Hi);
}
+/// The result is legal but the mask is not. Change the mask's element type to
+/// the result's integer element type, if every in-range index still fits.
+/// Out-of-range indices are poison, so it doesn't matter what they truncate to.
+SDValue DAGTypeLegalizer::LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "The source shares the result's type, so legalizing it "
+ "is a result legalization.");
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ EVT NewMaskVT = VT.changeVectorElementTypeToInteger();
+ // TODO: Scalable vectors need a bound on vscale to know the indices fit.
+ if (VT.isScalableVector() || NewMaskVT == N->getOperand(1).getValueType() ||
+ !TLI.isTypeLegal(NewMaskVT) ||
+ !isUIntN(NewMaskVT.getScalarSizeInBits(), VT.getVectorNumElements() - 1))
+ return TLI.expandVECTOR_SHUFFLE_VAR(N, DAG);
+
+ SDValue Mask = DAG.getZExtOrTrunc(N->getOperand(1), DL, NewMaskVT);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, VT, N->getOperand(0), Mask);
+}
+
SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
EVT ResVT = N->getValueType(0);
SDValue Lo, Hi;
@@ -5306,6 +5396,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
Res = WidenVecRes_VECTOR_COMPRESS(N);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = WidenVecRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::MLOAD:
Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
break;
@@ -6959,6 +7052,17 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
WideMask, WidePassthru);
}
+SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ // Only indices that were out of range can select the new source elements.
+ SDValue Src = GetWidenedVector(N->getOperand(0));
+ EVT WideVT = Src.getValueType();
+ EVT WideMaskVT = WideVT.changeVectorElementType(
+ *DAG.getContext(),
+ N->getOperand(1).getValueType().getVectorElementType());
+ SDValue Mask = ModifyToType(N->getOperand(1), WideMaskVT);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), WideVT, Src, Mask);
+}
+
SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
EVT VT = N->getValueType(0);
EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
@@ -7833,6 +7937,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::FLDEXP:
case ISD::FCOPYSIGN:
case ISD::LROUND:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7b1995f4fd7e37..d971bb1d5abbdd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7640,7 +7640,9 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
// We can't create a scalar CONCAT_VECTORS so skip it. It will break
// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
// foldCONCAT_VECTORS in getNode before this is called.
- if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
+ // VECTOR_SHUFFLE_VAR is not elementwise.
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS ||
+ Opcode == ISD::VECTOR_SHUFFLE_VAR)
return SDValue();
unsigned NumOps = Ops.size();
@@ -8829,6 +8831,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
}
break;
+ case ISD::VECTOR_SHUFFLE_VAR: {
+ [[maybe_unused]] EVT MaskVT = N2.getValueType();
+ assert(VT == N1.getValueType() &&
+ "VECTOR_SHUFFLE_VAR result and source types don't match.");
+ assert(VT.isVector() && MaskVT.isVector() &&
+ "VECTOR_SHUFFLE_VAR operands must be vectors.");
+ assert(MaskVT.getVectorElementType().isInteger() &&
+ "VECTOR_SHUFFLE_VAR mask must be a vector of integers.");
+ assert(VT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
+ "VECTOR_SHUFFLE_VAR mask has the wrong number of elements.");
+ break;
+ }
case ISD::EXTRACT_SUBVECTOR: {
EVT N1VT = N1.getValueType();
assert(VT.isVector() && N1VT.isVector() &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 065347d9030335..3b6a5c3a8de53e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8656,6 +8656,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
getValue(I.getArgOperand(1)),
getValue(I.getArgOperand(2)), Flags));
return;
+ case Intrinsic::vector_shuffle:
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, sdl,
+ TLI.getValueType(DAG.getDataLayout(), I.getType()),
+ getValue(I.getArgOperand(0)),
+ getValue(I.getArgOperand(1)), Flags));
+ return;
case Intrinsic::experimental_convergence_anchor:
case Intrinsic::experimental_convergence_entry:
case Intrinsic::experimental_convergence_loop:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8bc1707b47b314..c9f6a04c624737 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -474,6 +474,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::MGATHER: return "masked_gather";
case ISD::MSCATTER: return "masked_scatter";
case ISD::VECTOR_COMPRESS: return "vector_compress";
+ case ISD::VECTOR_SHUFFLE_VAR: return "vector_shuffle_var";
case ISD::VAARG: return "vaarg";
case ISD::VACOPY: return "vacopy";
case ISD::VAEND: return "vaend";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7c97dfb2c806a1..5598c826de35c9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13680,6 +13680,63 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
return DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo, Alignment);
}
+SDValue TargetLowering::expandVECTOR_SHUFFLE_VAR(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Node);
+ SDValue V = Node->getOperand(0);
+ SDValue Mask = Node->getOperand(1);
+
+ EVT VecVT = Node->getValueType(0);
+ EVT ScalarVT = VecVT.getScalarType();
+ EVT MaskScalarVT = Mask.getValueType().getScalarType();
+
+ // Needs to be handled by targets that have scalable vector types.
+ if (VecVT.isScalableVector())
+ report_fatal_error(
+ "Cannot expand vector_shuffle_var for scalable vectors.");
+
+ // Sub-byte elements can't be addressed on the stack, so shuffle a vector of
+ // wider integers instead.
+ if (!ScalarVT.isByteSized()) {
+ EVT WideVT;
+ for (MVT EltVT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+ WideVT = VecVT.changeVectorElementType(*DAG.getContext(), EltVT);
+ if (isTypeLegal(WideVT))
+ break;
+ }
+ SDValue Wide = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, V);
+ Wide = DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, WideVT, Wide, Mask);
+ return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Wide);
+ }
+
+ // Store V to the stack and load Res[i] from V[umin(Mask[i], NumElts - 1)].
+ // Out-of-range lanes are poison, so clamping them to the last element is
+ // fine.
+ Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
+ SDValue StackPtr = DAG.CreateStackTemporary(VecVT.getStoreSize(), Alignment);
+ auto &MF = DAG.getMachineFunction();
+ auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
+ auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
+
+ SDValue Chain =
+ DAG.getStore(DAG.getEntryNode(), DL, V, StackPtr, PtrInfo, Alignment);
+
+ // Freeze the mask so that poison or undef indices are still clamped.
+ Mask = DAG.getFreeze(Mask);
+
+ MVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
+ unsigned NumElts = VecVT.getVectorNumElements();
+ SmallVector<SDValue, 16> Elts;
+ Elts.reserve(NumElts);
+ for (unsigned I = 0; I < NumElts; ++I) {
+ SDValue Idx = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I);
+ Idx = DAG.getZExtOrTrunc(Idx, DL, IdxVT);
+ SDValue EltPtr = getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
+ Elts.push_back(DAG.getLoad(ScalarVT, DL, Chain, EltPtr, PtrInfo));
+ }
+ return DAG.getBuildVector(VecVT, DL, Elts);
+}
+
SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
SDLoc DL(Node);
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f852f7551ce16c..c689cebb77542c 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -951,7 +951,9 @@ void TargetLoweringBase::initActions() {
Expand);
// Only some target support these vector operations. Default them to Expand.
- setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
+ setOperationAction(
+ {ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH, ISD::VECTOR_SHUFFLE_VAR}, VT,
+ Expand);
// cttz.elts defaults to expand.
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1440b95896474f..3cdcb7acc2b5bb 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7057,6 +7057,14 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
}
break;
}
+ case Intrinsic::vector_shuffle: {
+ auto *RetTy = cast<VectorType>(Call.getType());
+ auto *MaskTy = cast<VectorType>(Call.getArgOperand(1)->getType());
+
+ Check(MaskTy->getElementCount() == RetTy->getElementCount(),
+ "Mask and return type must have the same number of elements.", &Call);
+ break;
+ }
case Intrinsic::vector_insert: {
Value *Vec = Call.getArgOperand(0);
Value *SubVec = Call.getArgOperand(1);
diff --git a/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll b/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
new file mode 100644
index 00000000000000..752f4732f7f686
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
@@ -0,0 +1,489 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown | FileCheck %s --check-prefixes=CHECK,LE
+; RUN: llc < %s -mtriple=aarch64_be-unknown | FileCheck %s --check-prefixes=CHECK,BE
+
+; There is no AArch64 lowering yet, so these all use the generic expansion.
+
+define <4 x i32> @shuffle_v4i32(<4 x i32> %v, <4 x i32> %mask) {
+; LE-LABEL: shuffle_v4i32:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: mov x11, sp
+; LE-NEXT: bfi x8, x10, #2, #2
+; LE-NEXT: mov w10, v1.s[2]
+; LE-NEXT: bfi x11, x9, #2, #2
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: mov w8, v1.s[3]
+; LE-NEXT: ld1 { v0.s }[1], [x11]
+; LE-NEXT: bfi x9, x10, #2, #2
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x8, #2, #2
+; LE-NEXT: ld1 { v0.s }[2], [x9]
+; LE-NEXT: ld1 { v0.s }[3], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v4i32:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4s, v1.4s
+; BE-NEXT: mov x8, sp
+; BE-NEXT: mov x13, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov w9, v1.s[2]
+; BE-NEXT: mov w10, v1.s[1]
+; BE-NEXT: fmov w12, s1
+; BE-NEXT: mov w11, v1.s[3]
+; BE-NEXT: bfi x8, x12, #2, #2
+; BE-NEXT: bfi x13, x9, #2, #2
+; BE-NEXT: mov x9, sp
+; BE-NEXT: bfi x9, x10, #2, #2
+; BE-NEXT: mov x10, sp
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: bfi x10, x11, #2, #2
+; BE-NEXT: ldr s1, [x13]
+; BE-NEXT: ld1 { v0.s }[1], [x9]
+; BE-NEXT: ld1 { v1.s }[1], [x10]
+; BE-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; BE-NEXT: rev64 v0.4s, v0.4s
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> %v, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <16 x i8> @shuffle_v16i8(<16 x i8> %v, <16 x i8> %mask) {
+; LE-LABEL: shuffle_v16i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: umov w10, v1.b[0]
+; LE-NEXT: umov w11, v1.b[8]
+; LE-NEXT: mov x13, sp
+; LE-NEXT: umov w9, v1.b[1]
+; LE-NEXT: umov w12, v1.b[9]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: umov w15, v1.b[11]
+; LE-NEXT: bfxil x13, x10, #0, #4
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfxil x8, x9, #0, #4
+; LE-NEXT: umov w9, v1.b[2]
+; LE-NEXT: umov w11, v1.b[10]
+; LE-NEXT: ldr b0, [x13]
+; LE-NEXT: bfxil x10, x12, #0, #4
+; LE-NEXT: ldr b2, [x14]
+; LE-NEXT: mov x12, sp
+; LE-NEXT: umov w13, v1.b[3]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: ld1 { v0.b }[1], [x8]
+; LE-NEXT: ld1 { v2.b }[1], [x10]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: bfxil x12, x9, #0, #4
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: umov w9, v1.b[4]
+; LE-NEXT: mov x10, sp
+; LE-NEXT: umov w11, v1.b[12]
+; LE-NEXT: ld1 { v0.b }[2], [x12]
+; LE-NEXT: ld1 { v2.b }[2], [x14]
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: bfxil x10, x15, #0, #4
+; LE-NEXT: mov x12, sp
+; LE-NEXT: umov w13, v1.b[5]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: umov w15, v1.b[13]
+; LE-NEXT: bfxil x12, x9, #0, #4
+; LE-NEXT: ld1 { v0.b }[3], [x8]
+; LE-NEXT: ld1 { v2.b }[3], [x10]
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: mov x8, sp
+; LE-NEXT: umov w9, v1.b[6]
+; LE-NEXT: mov x10, sp
+; LE-NEXT: umov w11, v1.b[14]
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: umov w13, v1.b[15]
+; LE-NEXT: ld1 { v0.b }[4], [x12]
+; LE-NEXT: ld1 { v2.b }[4], [x14]
+; LE-NEXT: bfxil x10, x15, #0, #4
+; LE-NEXT: umov w12, v1.b[7]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: mov x15, sp
+; LE-NEXT: bfxil x14, x9, #0, #4
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ld1 { v0.b }[5], [x8]
+; LE-NEXT: ld1 { v2.b }[5], [x10]
+; LE-NEXT: bfxil x15, x11, #0, #4
+; LE-NEXT: mov x8, sp
+; LE-NEXT: bfxil x9, x12, #0, #4
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: ld1 { v0.b }[6], [x14]
+; LE-NEXT: ld1 { v2.b }[6], [x15]
+; LE-NEXT: ld1 { v0.b }[7], [x9]
+; LE-NEXT: ld1 { v2.b }[7], [x8]
+; LE-NEXT: zip1 v0.2d, v0.2d, v2.2d
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v16i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.16b, v1.16b
+; BE-NEXT: mov x10, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: umov w8, v1.b[0]
+; BE-NEXT: umov w9, v1.b[1]
+; BE-NEXT: umov w11, v1.b[2]
+; BE-NEXT: bfxil x10, x8, #0, #4
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[3]
+; BE-NEXT: ldr b0, [x10]
+; BE-NEXT: umov w10, v1.b[4]
+; BE-NEXT: ld1 { v0.b }[1], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x11, #0, #4
+; BE-NEXT: ld1 { v0.b }[2], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[5]
+; BE-NEXT: ld1 { v0.b }[3], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[6]
+; BE-NEXT: ld1 { v0.b }[4], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[7]
+; BE-NEXT: ld1 { v0.b }[5], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[8]
+; BE-NEXT: ld1 { v0.b }[6], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[9]
+; BE-NEXT: ld1 { v0.b }[7], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[10]
+; BE-NEXT: ld1 { v0.b }[8], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[11]
+; BE-NEXT: ld1 { v0.b }[9], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[12]
+; BE-NEXT: ld1 { v0.b }[10], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[13]
+; BE-NEXT: ld1 { v0.b }[11], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[14]
+; BE-NEXT: ld1 { v0.b }[12], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[15]
+; BE-NEXT: ld1 { v0.b }[13], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: mov x10, sp
+; BE-NEXT: bfxil x10, x9, #0, #4
+; BE-NEXT: ld1 { v0.b }[14], [x8]
+; BE-NEXT: ld1 { v0.b }[15], [x10]
+; BE-NEXT: rev64 v0.16b, v0.16b
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <16 x i8> @llvm.vector.shuffle.v16i8.v16i8(<16 x i8> %v, <16 x i8> %mask)
+ ret <16 x i8> %res
+}
+
+define <2 x i64> @shuffle_v2i64(<2 x i64> %v, <2 x i64> %mask) {
+; LE-LABEL: shuffle_v2i64:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov x10, d1
+; LE-NEXT: mov x9, v1.d[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: bfi x8, x10, #3, #1
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x9, #3, #1
+; LE-NEXT: ldr d0, [x8]
+; LE-NEXT: ld1 { v0.d }[1], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v2i64:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov x8, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: fmov x10, d1
+; BE-NEXT: mov x9, v1.d[1]
+; BE-NEXT: bfi x8, x10, #3, #1
+; BE-NEXT: mov x10, sp
+; BE-NEXT: bfi x10, x9, #3, #1
+; BE-NEXT: ldr d0, [x8]
+; BE-NEXT: ld1 { v0.d }[1], [x10]
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <2 x i64> @llvm.vector.shuffle.v2i64.v2i64(<2 x i64> %v, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x float> @shuffle_v4f32(<4 x float> %v, <4 x i32> %mask) {
+; LE-LABEL: shuffle_v4f32:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: mov x11, sp
+; LE-NEXT: bfi x8, x10, #2, #2
+; LE-NEXT: mov w10, v1.s[2]
+; LE-NEXT: bfi x11, x9, #2, #2
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: mov w8, v1.s[3]
+; LE-NEXT: ld1 { v0.s }[1], [x11]
+; LE-NEXT: bfi x9, x10, #2, #2
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x8, #2, #2
+; LE-NEXT: ld1 { v0.s }[2], [x9]
+; LE-NEXT: ld1 { v0.s }[3], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v4f32:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4s, v1.4s
+; BE-NEXT: mov x8, sp
+; BE-NEXT: mov x13, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov w9, v1.s[2]
+; BE-NEXT: mov w10, v1.s[1]
+; BE-NEXT: fmov w12, s1
+; BE-NEXT: mov w11, v1.s[3]
+; BE-NEXT: bfi x8, x12, #2, #2
+; BE-NEXT: bfi x13, x9, #2, #2
+; BE-NEXT: mov x9, sp
+; BE-NEXT: bfi x9, x10, #2, #2
+; BE-NEXT: mov x10, sp
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: bfi x10, x11, #2, #2
+; BE-NEXT: ldr s1, [x13]
+; BE-NEXT: ld1 { v0.s }[1], [x9]
+; BE-NEXT: ld1 { v1.s }[1], [x10]
+; BE-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; BE-NEXT: rev64 v0.4s, v0.4s
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x float> @llvm.vector.shuffle.v4f32.v4i32(<4 x float> %v, <4 x i32> %mask)
+ ret <4 x float> %res
+}
+
+; The mask is promoted with a zero extension.
+define <4 x i16> @promote_mask_v4i16_v4i8(<4 x i16> %v, <4 x i8> %mask) {
+; LE-LABEL: promote_mask_v4i16_v4i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bic v1.4h, #255, lsl #8
+; LE-NEXT: umov w8, v1.h[0]
+; LE-NEXT: umov w9, v1.h[1]
+; LE-NEXT: umov w11, v1.h[2]
+; LE-NEXT: bfi x10, x8, #1, #2
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x9, #1, #2
+; LE-NEXT: umov w9, v1.h[3]
+; LE-NEXT: ldr h0, [x10]
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: ld1 { v0.h }[1], [x8]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x11, #1, #2
+; LE-NEXT: bfi x10, x9, #1, #2
+; LE-NEXT: ld1 { v0.h }[2], [x8]
+; LE-NEXT: ld1 { v0.h }[3], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_mask_v4i16_v4i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4h, v1.4h
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: bic v1.4h, #255, lsl #8
+; BE-NEXT: umov w8, v1.h[0]
+; BE-NEXT: umov w9, v1.h[1]
+; BE-NEXT: umov w11, v1.h[2]
+; BE-NEXT: bfi x10, x8, #1, #2
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x9, #1, #2
+; BE-NEXT: umov w9, v1.h[3]
+; BE-NEXT: ldr h0, [x10]
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: ld1 { v0.h }[1], [x8]
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x11, #1, #2
+; BE-NEXT: bfi x10, x9, #1, #2
+; BE-NEXT: ld1 { v0.h }[2], [x8]
+; BE-NEXT: ld1 { v0.h }[3], [x10]
+; BE-NEXT: rev64 v0.4h, v0.4h
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i16> @llvm.vector.shuffle.v4i16.v4i8(<4 x i16> %v, <4 x i8> %mask)
+ ret <4 x i16> %res
+}
+
+; Both the result and the mask are promoted.
+define <4 x i8> @promote_res_v4i8(<4 x i8> %v, <4 x i8> %mask) {
+; LE-LABEL: promote_res_v4i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bic v1.4h, #255, lsl #8
+; LE-NEXT: umov w8, v1.h[0]
+; LE-NEXT: umov w9, v1.h[1]
+; LE-NEXT: umov w11, v1.h[2]
+; LE-NEXT: bfi x10, x8, #1, #2
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x9, #1, #2
+; LE-NEXT: umov w9, v1.h[3]
+; LE-NEXT: ldr h0, [x10]
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: ld1 { v0.h }[1], [x8]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x11, #1, #2
+; LE-NEXT: bfi x10, x9, #1, #2
+; LE-NEXT: ld1 { v0.h }[2], [x8]
+; LE-NEXT: ld1 { v0.h }[3], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_res_v4i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4h, v1.4h
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: bic v1.4h, #255, lsl #8
+; BE-NEXT: umov w8, v1.h[0]
+; BE-NEXT: umov w9, v1.h[1]
+; BE-NEXT: umov w11, v1.h[2]
+; BE-NEXT: bfi x10, x8, #1, #2
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x9, #1, #2
+; BE-NEXT: umov w9, v1.h[3]
+; BE-NEXT: ldr h0, [x10]
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: ld1 { v0.h }[1], [x8]
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x11, #1, #2
+; BE-NEXT: bfi x10, x9, #1, #2
+; BE-NEXT: ld1 { v0.h }[2], [x8]
+; BE-NEXT: ld1 { v0.h }[3], [x10]
+; BE-NEXT: rev64 v0.4h, v0.4h
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i8> @llvm.vector.shuffle.v4i8.v4i8(<4 x i8> %v, <4 x i8> %mask)
+ ret <4 x i8> %res
+}
+
+define <2 x i16> @promote_res_v2i16(<2 x i16> %v, <2 x i16> %mask) {
+; LE-LABEL: promote_res_v2i16:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bfi x8, x10, #2, #1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: bfi x10, x9, #2, #1
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: ld1 { v0.s }[1], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_res_v2i16:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.2s, v1.2s
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: fmov w10, s1
+; BE-NEXT: mov w9, v1.s[1]
+; BE-NEXT: bfi x8, x10, #2, #1
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: bfi x10, x9, #2, #1
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: ld1 { v0.s }[1], [x10]
+; BE-NEXT: rev64 v0.2s, v0.2s
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <2 x i16> @llvm.vector.shuffle.v2i16.v2i16(<2 x i16> %v, <2 x i16> %mask)
+ ret <2 x i16> %res
+}
+
+; v1i64 is legal, so this is expanded rather than scalarized.
+define <1 x i64> @shuffle_v1i64(<1 x i64> %v, <1 x i64> %mask) {
+; CHECK-LABEL: shuffle_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: ret
+ %res = call <1 x i64> @llvm.vector.shuffle.v1i64.v1i64(<1 x i64> %v, <1 x i64> %mask)
+ ret <1 x i64> %res
+}
+
+; The result is legal but the mask is scalarized.
+define <1 x i64> @shuffle_v1i64_v1i1(<1 x i64> %v, <1 x i1> %mask) {
+; CHECK-LABEL: shuffle_v1i64_v1i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
+ %res = call <1 x i64> @llvm.vector.shuffle.v1i64.v1i1(<1 x i64> %v, <1 x i1> %mask)
+ ret <1 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll b/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
new file mode 100644
index 00000000000000..3b2ee665ff233e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; There is no X86 lowering yet, so these all use the generic expansion.
+
+define <4 x i32> @shuffle_v4i32(<4 x i32> %v, <4 x i32> %mask) {
+; SSE2-LABEL: shuffle_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> %v, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <2 x i64> @shuffle_v2i64(<2 x i64> %v, <2 x i64> %mask) {
+; SSE2-LABEL: shuffle_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: andl $1, %eax
+; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: andl $1, %eax
+; SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX2-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT: vmovq %xmm1, %rax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: retq
+ %res = call <2 x i64> @llvm.vector.shuffle.v2i64.v2i64(<2 x i64> %v, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+; Must not be constant folded elementwise.
+define <4 x i32> @shuffle_v4i32_const() {
+; SSE2-LABEL: shuffle_v4i32_const:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps {{.*#+}} xmm0 = [40,30,20,10]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v4i32_const:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps {{.*#+}} xmm0 = [40,30,20,10]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v4i32_const:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm0 = [40,30,20,10]
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> <i32 10, i32 20, i32 30, i32 40>, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <4 x i32> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll b/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
new file mode 100644
index 00000000000000..faf725922374cd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
@@ -0,0 +1,1155 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; Type legalization of VECTOR_SHUFFLE_VAR. There is no X86 lowering yet, so
+; the legalized nodes use the generic expansion.
+
+; The only in-range index is 0, so the result is the source.
+define <1 x i32> @scalarize_v1i32(<1 x i32> %v, <1 x i32> %mask) {
+; CHECK-LABEL: scalarize_v1i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: retq
+ %res = call <1 x i32> @llvm.vector.shuffle.v1i32.v1i32(<1 x i32> %v, <1 x i32> %mask)
+ ret <1 x i32> %res
+}
+
+; The result is widened to v4i32 and the mask with it.
+define <3 x i32> @widen_v3i32(<3 x i32> %v, <3 x i32> %mask) {
+; SSE2-LABEL: widen_v3i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: widen_v3i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: widen_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <3 x i32> @llvm.vector.shuffle.v3i32.v3i32(<3 x i32> %v, <3 x i32> %mask)
+ ret <3 x i32> %res
+}
+
+; The result is widened to v4i32. The mask is widened and then changed to
+; v4i32.
+define <2 x i32> @widen_v2i32_v2i8(<2 x i32> %v, <2 x i8> %mask) {
+; SSE2-LABEL: widen_v2i32_v2i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: pextrw $0, %xmm1, %eax
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: widen_v2i32_v2i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrb $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: widen_v2i32_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrb $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <2 x i32> @llvm.vector.shuffle.v2i32.v2i8(<2 x i32> %v, <2 x i8> %mask)
+ ret <2 x i32> %res
+}
+
+; Without AVX512, each result half shuffles both source halves and selects
+; between them.
+define <8 x i32> @split_v8i32(<8 x i32> %v, <8 x i32> %mask) {
+; SSE2-LABEL: split_v8i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4,4,4,4]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[3,3,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,1,1,1]
+; SSE2-NEXT: psubd %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm8, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm9[0]
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT: pxor %xmm4, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483652,2147483652,2147483652,2147483652]
+; SSE2-NEXT: movdqa %xmm2, %xmm9
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm9
+; SSE2-NEXT: movd %xmm5, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm6, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSE2-NEXT: pand %xmm9, %xmm0
+; SSE2-NEXT: pandn %xmm8, %xmm9
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: pxor %xmm3, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
+; SSE2-NEXT: movd %xmm8, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm5, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm6, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm9 = xmm9[0],xmm8[0]
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE2-NEXT: pand %xmm2, %xmm1
+; SSE2-NEXT: pandn %xmm9, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm0, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm0, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i32(<8 x i32> %v, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+; Split with a mask element type that differs from the result's.
+define <8 x i32> @split_v8i32_v8i8(<8 x i32> %v, <8 x i8> %mask) {
+; SSE2-LABEL: split_v8i32_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [4,4,4,4,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pmaxub %xmm3, %xmm5
+; SSE2-NEXT: pcmpeqb %xmm2, %xmm5
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm2, %eax
+; SSE2-NEXT: movdqa %xmm2, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; SSE2-NEXT: movd %xmm7, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; SSE2-NEXT: movd %xmm6, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm7[0]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psubb %xmm3, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; SSE2-NEXT: movd %xmm7, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; SSE2-NEXT: movd %xmm6, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm7[0]
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: movdqa %xmm4, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm2, %eax
+; SSE2-NEXT: movdqa %xmm2, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa %xmm4, %xmm8
+; SSE2-NEXT: psubb %xmm3, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm4, %eax
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm4[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [4,4,4,4,4,4,4,4,u,u,u,u,u,u,u,u]
+; SSE2-NEXT: pmaxub %xmm8, %xmm5
+; SSE2-NEXT: pcmpeqb %xmm8, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: pand %xmm5, %xmm1
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: por %xmm5, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm0, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm0, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i8(<8 x i32> %v, <8 x i8> %mask)
+ ret <8 x i32> %res
+}
+
+; An i2 index can't reach the high half, so only the low half is shuffled.
+define <8 x i32> @split_v8i32_v8i2(<8 x i32> %v, <8 x i2> %mask) {
+; SSE2-LABEL: split_v8i32_v8i2:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [3,3,3,3]
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i2:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i2(<8 x i32> %v, <8 x i2> %mask)
+ ret <8 x i32> %res
+}
+
+; Same with an i1 index.
+define <8 x i32> @split_v8i32_v8i1(<8 x i32> %v, <8 x i1> %mask) {
+; SSE2-LABEL: split_v8i32_v8i1:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,1,1,1]
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1
+; AVX512-NEXT: vpmovw2m %xmm1, %k3
+; AVX512-NEXT: kshiftrw $3, %k3, %k0
+; AVX512-NEXT: kshiftrw $2, %k3, %k1
+; AVX512-NEXT: kshiftrw $1, %k3, %k2
+; AVX512-NEXT: kshiftrw $7, %k3, %k4
+; AVX512-NEXT: kshiftrw $6, %k3, %k5
+; AVX512-NEXT: kshiftrw $4, %k3, %k6
+; AVX512-NEXT: kshiftrw $5, %k3, %k7
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: kmovd %k7, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: kmovd %k6, %ecx
+; AVX512-NEXT: andl $1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k5, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k4, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k3, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-NEXT: kmovd %k2, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: kmovd %k1, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i1(<8 x i32> %v, <8 x i1> %mask)
+ ret <8 x i32> %res
+}
+
+; The result is legal but the mask would be split, so it is truncated to v4i32
+; instead.
+define <4 x i32> @mask_split_v4i32_v4i64(<4 x i32> %v, <4 x i64> %mask) {
+; SSE2-LABEL: mask_split_v4i32_v4i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: mask_split_v4i32_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovq %xmm1, %rcx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: mask_split_v4i32_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovq %xmm1, %rcx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vmovq %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i64(<4 x i32> %v, <4 x i64> %mask)
+ ret <4 x i32> %res
+}
+
+define <16 x i8> @mask_split_v16i8_v16i16(<16 x i8> %v, <16 x i16> %mask) {
+; SSE2-LABEL: mask_split_v16i8_v16i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: packuswb %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: mask_split_v16i8_v16i16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: movzbl -24(%rsp,%rcx), %ecx
+; AVX2-NEXT: vmovd %ecx, %xmm0
+; AVX2-NEXT: vpinsrb $1, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $2, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $3, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $4, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $4, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $5, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $6, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $7, -24(%rsp,%rax), %xmm0, %xmm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $8, -24(%rsp,%rax), %xmm2, %xmm1
+; AVX2-NEXT: vpextrw $1, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $9, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $2, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $10, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $3, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $11, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $4, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $12, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $5, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $13, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $6, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $14, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $7, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $15, -24(%rsp,%rax), %xmm1, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: mask_split_v16i8_v16i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $15, %ecx
+; AVX512-NEXT: movzbl -24(%rsp,%rcx), %ecx
+; AVX512-NEXT: vmovd %ecx, %xmm0
+; AVX512-NEXT: vpinsrb $1, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $2, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $3, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $4, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $4, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $5, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $6, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $7, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vmovd %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $8, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $9, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $10, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $11, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $4, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $12, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $13, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $14, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $15, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %res = call <16 x i8> @llvm.vector.shuffle.v16i8.v16i16(<16 x i8> %v, <16 x i16> %mask)
+ ret <16 x i8> %res
+}
+
+; With AVX512 the result is legal but the mask isn't. The mask can't be
+; truncated to i1, which can't hold the in-range indices.
+define <8 x i1> @shuffle_v8i1_v8i8(<8 x i1> %v, <8 x i8> %mask) {
+; SSE2-LABEL: shuffle_v8i1_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: pextrw $7, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pextrw $6, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT: pextrw $5, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pextrw $4, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: pextrw $3, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pextrw $2, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT: pextrw $1, %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $7, %eax
+; SSE2-NEXT: movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v8i1_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrb $1, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
+; AVX2-NEXT: vmovd %ecx, %xmm0
+; AVX2-NEXT: vpinsrw $1, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $2, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $2, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $3, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $3, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $4, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $4, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $5, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $5, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $6, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $6, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $7, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrw $7, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v8i1_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrb $1, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: movzwl -24(%rsp,%rcx,2), %ecx
+; AVX512-NEXT: vmovd %ecx, %xmm0
+; AVX512-NEXT: vpinsrw $1, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $2, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $2, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $3, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $3, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $4, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $4, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $5, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $5, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $6, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $6, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $7, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrw $7, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <8 x i1> @llvm.vector.shuffle.v8i1.v8i8(<8 x i1> %v, <8 x i8> %mask)
+ ret <8 x i1> %res
+}
diff --git a/llvm/test/Verifier/vector-shuffle-var.ll b/llvm/test/Verifier/vector-shuffle-var.ll
new file mode 100644
index 00000000000000..f43bf61f78e696
--- /dev/null
+++ b/llvm/test/Verifier/vector-shuffle-var.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes=verify -S < %s 2>&1 | FileCheck %s
+
+; CHECK: Mask and return type must have the same number of elements.
+define <4 x i32> @mask_too_long(<4 x i32> %v, <8 x i8> %mask) {
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v8i8(<4 x i32> %v, <8 x i8> %mask)
+ ret <4 x i32> %res
+}
+
+; CHECK: Mask and return type must have the same number of elements.
+define <4 x i32> @mask_too_short(<4 x i32> %v, <2 x i8> %mask) {
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v2i8(<4 x i32> %v, <2 x i8> %mask)
+ ret <4 x i32> %res
+}
+
+; CHECK: Mask and return type must have the same number of elements.
+define <vscale x 4 x i32> @mixed_scalable(<vscale x 4 x i32> %v, <4 x i8> %mask) {
+ %res = call <vscale x 4 x i32> @llvm.vector.shuffle.nxv4i32.v4i8(<vscale x 4 x i32> %v, <4 x i8> %mask)
+ ret <vscale x 4 x i32> %res
+}
More information about the llvm-commits
mailing list