[llvm] [IR] Add llvm.vector.shuffle intrinsic for vector shuffles with runtime masks Part 1. (PR #219259)
Oscar Smith via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 22:11:23 PDT 2026
https://github.com/oscardssmith updated https://github.com/llvm/llvm-project/pull/219259
>From 89c157183ea3fb7b8cbf0f3854d9f0e3b80081a7 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscar.smith at juliahub.com>
Date: Wed, 23 Sep 2026 05:11:08 +0000
Subject: [PATCH] [IR][SelectionDAG] Add llvm.vector.shuffle intrinsic
Add an intrinsic that permutes a vector by a mask that need not be a
constant:
<N x ty> @llvm.vector.shuffle(<N x ty> %vec, <N x iK> %mask)
Result element i is %vec[zext(%mask[i])], or poison if that index is out
of range. Poison is the one out-of-range behavior that x86 VPERMV/PSHUFB,
AArch64 TBL and RISC-V vrgather can all implement without fixup code, and
it lets the intrinsic be speculatable. The mask element type is overloaded
so targets can take their native index width directly.
This follows the RFC at
https://discourse.llvm.org/t/rfc-ir-ability-to-shuffle-vectors-with-dynamic-mask/91282
and is the first part of #208537. It adds the intrinsic, its verifier
check, and the ISD::VECTOR_SHUFFLE_VAR node with type legalization and a
generic expansion through a stack temporary. No target lowers the node
natively yet, and scalable vectors are not supported until one does.
Target lowering, the cost model and combines will follow separately.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/docs/LangRef.md | 37 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 9 +
llvm/include/llvm/CodeGen/TargetLowering.h | 5 +
llvm/include/llvm/IR/Intrinsics.td | 5 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 23 +
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 6 +
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 119 ++
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 17 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 +
.../SelectionDAG/SelectionDAGDumper.cpp | 1 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 47 +
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 +-
llvm/lib/IR/Verifier.cpp | 11 +
.../AArch64/vector-shuffle-var-expand.ll | 485 ++++++++
.../CodeGen/X86/vector-shuffle-var-expand.ll | 132 +++
.../X86/vector-shuffle-var-legalize.ll | 1048 +++++++++++++++++
llvm/test/Verifier/vector-shuffle-var.ll | 20 +
18 files changed, 1977 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
create mode 100644 llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
create mode 100644 llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
create mode 100644 llvm/test/Verifier/vector-shuffle-var.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index a4816d337e02b3..f3bed2d190f2c3 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -21489,6 +21489,43 @@ VecT compress(VecT vec, VecT mask, VecT passthru) {
}
```
+(int_vector_shuffle)=
+
+#### '`llvm.vector.shuffle.*`' Intrinsic
+
+##### Syntax:
+
+This is an overloaded intrinsic.
+
+```llvm
+declare <8 x i32> @llvm.vector.shuffle.v8i32.v8i8(<8 x i32> %vec, <8 x i8> %mask)
+declare <vscale x 4 x float> @llvm.vector.shuffle.nxv4f32.nxv4i16(<vscale x 4 x float> %vec, <vscale x 4 x i16> %mask)
+```
+
+##### Overview:
+
+The '`llvm.vector.shuffle.*`' intrinsics permute the elements of a vector
+according to a mask that does not need to be a constant.
+
+For example:
+
+```text
+llvm.vector.shuffle(<A,B,C,D>, <3,0,0,7>) ==> <D,A,A,poison>
+```
+
+##### Arguments:
+
+The first operand is the vector to permute. The result has the same type. The
+second operand is the mask, a vector of integers with the same number of
+elements as the first operand.
+
+##### Semantics:
+
+Element `i` of the result is element `%mask[i]` of `%vec`, where `%mask[i]` is
+treated as an unsigned index. If `%mask[i]` is poison, or is greater than or
+equal to the number of elements in `%vec`, element `i` of the result is a
+{ref}`poison value <poisonvalues>`.
+
#### '`llvm.experimental.vector.match.*`' Intrinsic
##### Syntax:
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 1af501d4c8dd05..daa748c18982ba 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -701,6 +701,15 @@ enum NodeType {
/// If passthru is undef, ?s remain undefined.
VECTOR_COMPRESS,
+ /// VECTOR_SHUFFLE_VAR(VEC, MASK) - Like VECTOR_SHUFFLE, but with a single
+ /// source and with the mask a run-time integer vector operand rather than
+ /// an array of constants. The result has VEC's type, and MASK has the same
+ /// element count as VEC but an arbitrary integer element type. Each mask
+ /// element is an unsigned index into VEC:
+ /// RESULT[i] = VEC[zext(MASK[i])]
+ /// if zext(MASK[i]) < VEC.ElementCount, otherwise RESULT[i] is poison.
+ VECTOR_SHUFFLE_VAR,
+
/// MULHU/MULHS - Multiply high - Multiply two integers of type iN,
/// producing an unsigned/signed value of type i[2*N], then return the top
/// part.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c08d0e53ec34b3..2f758622fa5592 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -6053,6 +6053,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// temporarily, advance store position, before re-loading the final vector.
SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const;
+ /// Expand a VECTOR_SHUFFLE_VAR by storing the source vector to a stack
+ /// temporary and loading each result element through a variable-indexed
+ /// (clamped) element pointer.
+ SDValue expandVECTOR_SHUFFLE_VAR(SDNode *Node, SelectionDAG &DAG) const;
+
/// Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for
/// each active lane (i), getting the maximum and subtracting it from VL.
SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 26b7c772eb2446..5b15d1303d867e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2790,6 +2790,11 @@ def int_vector_splice_right
[LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
[IntrNoMem, IntrSpeculatable]>;
+def int_vector_shuffle:
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [LLVMMatchType<0>, llvm_any_vector_int_ty],
+ [IntrNoMem, IntrSpeculatable]>;
+
//===---------- Intrinsics to query properties of scalable vectors --------===//
def int_vscale : DefaultAttrsIntrinsic<[llvm_anyint_ty],
[],
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index a90bb06fb424dc..66d0c6bbae46ff 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -90,6 +90,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
Res = PromoteIntRes_VECTOR_COMPRESS(N);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = PromoteIntRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::SELECT:
case ISD::VSELECT:
case ISD::VP_MERGE:
@@ -1039,6 +1042,14 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(SDNode *N) {
N->getOperand(1), Passthru);
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ // The source has the same type as the result, so promoting the result just
+ // means shuffling the promoted source. The mask is untouched.
+ SDValue V = GetPromotedInteger(N->getOperand(0));
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), V.getValueType(), V,
+ N->getOperand(1));
+}
+
/// Promote the overflow flag of an overflowing arithmetic node.
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
// Change the return type of the boolean result while obeying
@@ -2050,6 +2061,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_COMPRESS:
Res = PromoteIntOp_VECTOR_COMPRESS(N, OpNo);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = PromoteIntOp_VECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
case ISD::BF16_TO_FP:
case ISD::FP16_TO_FP:
@@ -2633,6 +2647,15 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_COMPRESS(SDNode *N,
return DAG.getNode(ISD::VECTOR_COMPRESS, SDLoc(N), VT, Vec, Mask, Passthru);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_SHUFFLE_VAR(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "The source shares the result's type, so promoting it "
+ "is a result promotion.");
+ // Mask elements are unsigned indices, so they must be zero-extended.
+ SDValue Mask = ZExtPromotedInteger(N->getOperand(1));
+ return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Mask), 0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index c9583d99ddbf43..18658dd0d12e37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -317,6 +317,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
SDValue PromoteIntRes_VECTOR_COMPRESS(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue PromoteIntRes_Overflow(SDNode *N);
SDValue PromoteIntRes_FFREXP(SDNode *N);
SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
@@ -402,6 +403,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
SDValue PromoteIntOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);
+ SDValue PromoteIntOp_VECTOR_SHUFFLE_VAR(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
SDValue PromoteIntOp_FIX(SDNode *N);
SDValue PromoteIntOp_ExpOp(SDNode *N);
@@ -837,6 +839,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_BITCAST(SDNode *N);
SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
+ SDValue ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
SDValue ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N);
SDValue ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N);
@@ -949,6 +952,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_Gather(MemSDNode *VPGT, SDValue &Lo, SDValue &Hi,
bool SplitSETCC = false);
void SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -976,6 +980,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_UnaryOp(SDNode *N);
SDValue SplitVecOp_TruncateHelper(SDNode *N);
SDValue SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);
+ SDValue LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_BITCAST(SDNode *N);
SDValue SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
@@ -1057,6 +1062,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N);
SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);
SDValue WidenVecRes_VECTOR_COMPRESS(SDNode *N);
+ SDValue WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N);
SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
SDValue WidenVecRes_VP_GATHER(VPGatherSDNode* N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 5e3f252fdd3d48..26ecf97b9231e0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -479,6 +479,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::FP_TO_UINT_SAT:
case ISD::MGATHER:
case ISD::VECTOR_COMPRESS:
+ case ISD::VECTOR_SHUFFLE_VAR:
case ISD::SCMP:
case ISD::UCMP:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
@@ -1303,6 +1304,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECTOR_COMPRESS:
Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG));
return;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Results.push_back(TLI.expandVECTOR_SHUFFLE_VAR(Node, DAG));
+ return;
case ISD::CTTZ_ELTS:
case ISD::CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandCttzElts(Node, DAG));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 56ad7c65b48d1b..3d23823d7c01df 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -65,6 +65,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ R = ScalarizeVecRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
case ISD::CONVERT_FROM_ARBITRARY_FP:
R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
@@ -491,6 +494,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
N->getOperand(0), N->getOperand(1));
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ // The source has one element, so index 0 is the only one in range; every
+ // other index is out of range and yields poison, which that element refines.
+ return GetScalarizedVector(N->getOperand(0));
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
SDLoc DL(N);
SDValue Op = N->getOperand(0);
@@ -1446,6 +1455,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ SplitVecRes_VECTOR_SHUFFLE_VAR(N, Lo, Hi);
+ break;
case ISD::SETCC:
SplitVecRes_SETCC(N, Lo, Hi);
break;
@@ -2928,6 +2940,69 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
std::tie(Lo, Hi) = DAG.SplitVector(Compressed, DL);
}
+void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ SDLoc DL(N);
+ SDValue Src = N->getOperand(0), Mask = N->getOperand(1);
+ EVT MaskVT = Mask.getValueType();
+
+ // Choosing a mask type that can hold a scalable HalfElts needs a bound on
+ // vscale, which belongs with the first native scalable lowering. Until then
+ // no target can lower the halves anyway.
+ if (MaskVT.isScalableVector())
+ report_fatal_error("Cannot split scalable vector_shuffle_var.");
+
+ // The source has the result's type, so it is always being split too.
+ SDValue SrcLo, SrcHi;
+ GetSplitVector(Src, SrcLo, SrcHi);
+
+ SDValue MaskLo, MaskHi;
+ if (getTypeAction(MaskVT) == TargetLowering::TypeSplitVector)
+ GetSplitVector(Mask, MaskLo, MaskHi);
+ else
+ std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
+
+ EVT HalfVT = SrcLo.getValueType();
+ EVT HalfMaskVT = MaskLo.getValueType();
+ unsigned HalfElts = HalfVT.getVectorNumElements();
+
+ // Every index the mask element type can hold lands in the low half of the
+ // source, so the high half is unreachable and each result half is just a
+ // shuffle of the low source half.
+ if (APInt::getMaxValue(MaskVT.getScalarSizeInBits()).ult(HalfElts)) {
+ Lo = DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, MaskLo);
+ Hi = DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, MaskHi);
+ return;
+ }
+
+ // Otherwise shuffle both source halves with the same indices and select per
+ // lane on which half the index falls in:
+ // Half[i] = Idx[i] <u HalfElts ? SrcLo[Idx[i]] : SrcHi[Idx[i] - HalfElts]
+ // Indices past the end of the whole source are out of range in the SrcHi
+ // shuffle too, so they stay poison. HalfElts is representable in the mask
+ // element type, since the early return above took every case where it is
+ // not.
+ SDValue HalfEltsV = DAG.getConstant(HalfElts, DL, HalfMaskVT);
+ // The compare is done in the mask type, but the select's condition has to be
+ // whatever the target uses for selects on the result type.
+ EVT CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ HalfMaskVT);
+ EVT CCVT =
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HalfVT);
+ auto ShuffleHalf = [&](SDValue Idx) {
+ SDValue FromLo =
+ DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcLo, Idx);
+ SDValue HiIdx = DAG.getNode(ISD::SUB, DL, HalfMaskVT, Idx, HalfEltsV);
+ SDValue FromHi =
+ DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, HalfVT, SrcHi, HiIdx);
+ SDValue InLo = DAG.getSetCC(DL, CmpVT, Idx, HalfEltsV, ISD::SETULT);
+ InLo = DAG.getBoolExtOrTrunc(InLo, DL, CCVT, HalfMaskVT);
+ return DAG.getSelect(DL, HalfVT, InLo, FromLo, FromHi);
+ };
+ Lo = ShuffleHalf(MaskLo);
+ Hi = ShuffleHalf(MaskHi);
+}
+
void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
assert(N->getValueType(0).isVector() &&
N->getOperand(0).getValueType().isVector() &&
@@ -3850,6 +3925,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_COMPRESS:
Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::STRICT_SINT_TO_FP:
case ISD::STRICT_UINT_TO_FP:
case ISD::SINT_TO_FP:
@@ -4073,6 +4151,27 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VecVT, Lo, Hi);
}
+/// The mask's element count is pinned to the (legal) result's, so an illegal
+/// mask type can only be fixed by changing its element type. Retry with the
+/// result's integer element type, which is legal whenever the result type is.
+/// Indices are unsigned and out-of-range lanes are poison, so narrowing the
+/// index type is always a legal refinement. This handles both a split and a
+/// widened mask, since either way the fix is the same re-typing.
+SDValue DAGTypeLegalizer::LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "The source shares the result's type, so legalizing it "
+ "is a result legalization.");
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ EVT NewMaskVT = VT.changeVectorElementTypeToInteger();
+ if (NewMaskVT == N->getOperand(1).getValueType() ||
+ !TLI.isTypeLegal(NewMaskVT))
+ return TLI.expandVECTOR_SHUFFLE_VAR(N, DAG);
+
+ SDValue Mask = DAG.getZExtOrTrunc(N->getOperand(1), DL, NewMaskVT);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, DL, VT, N->getOperand(0), Mask);
+}
+
SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
EVT ResVT = N->getValueType(0);
SDValue Lo, Hi;
@@ -5306,6 +5405,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_COMPRESS:
Res = WidenVecRes_VECTOR_COMPRESS(N);
break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = WidenVecRes_VECTOR_SHUFFLE_VAR(N);
+ break;
case ISD::MLOAD:
Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
break;
@@ -6959,6 +7061,20 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
WideMask, WidePassthru);
}
+SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
+ // The source has the result's type, so it is widened to the same type. The
+ // appended source lanes are undef, but only indices that were out of range
+ // (and therefore poison) can select them. The mask is widened to match, and
+ // its undef lanes make the extra result lanes poison.
+ SDValue Src = GetWidenedVector(N->getOperand(0));
+ EVT WideVT = Src.getValueType();
+ EVT WideMaskVT = WideVT.changeVectorElementType(
+ *DAG.getContext(),
+ N->getOperand(1).getValueType().getVectorElementType());
+ SDValue Mask = ModifyToType(N->getOperand(1), WideMaskVT);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, SDLoc(N), WideVT, Src, Mask);
+}
+
SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
EVT VT = N->getValueType(0);
EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
@@ -7833,6 +7949,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
+ case ISD::VECTOR_SHUFFLE_VAR:
+ Res = LegalizeMaskTypeForVECTOR_SHUFFLE_VAR(N, OpNo);
+ break;
case ISD::FLDEXP:
case ISD::FCOPYSIGN:
case ISD::LROUND:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7b1995f4fd7e37..230b68e5361c35 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7640,7 +7640,10 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
// We can't create a scalar CONCAT_VECTORS so skip it. It will break
// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
// foldCONCAT_VECTORS in getNode before this is called.
- if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
+ // VECTOR_SHUFFLE_VAR is not elementwise: result lane I does not come from
+ // lane I of its source.
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS ||
+ Opcode == ISD::VECTOR_SHUFFLE_VAR)
return SDValue();
unsigned NumOps = Ops.size();
@@ -8829,6 +8832,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
}
break;
+ case ISD::VECTOR_SHUFFLE_VAR: {
+ [[maybe_unused]] EVT MaskVT = N2.getValueType();
+ assert(VT == N1.getValueType() &&
+ "VECTOR_SHUFFLE_VAR result and source types don't match.");
+ assert(VT.isVector() && MaskVT.isVector() &&
+ "VECTOR_SHUFFLE_VAR operands must be vectors.");
+ assert(MaskVT.getVectorElementType().isInteger() &&
+ "VECTOR_SHUFFLE_VAR mask must be a vector of integers.");
+ assert(VT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
+ "VECTOR_SHUFFLE_VAR mask has the wrong number of elements.");
+ break;
+ }
case ISD::EXTRACT_SUBVECTOR: {
EVT N1VT = N1.getValueType();
assert(VT.isVector() && N1VT.isVector() &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 065347d9030335..3b6a5c3a8de53e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8656,6 +8656,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
getValue(I.getArgOperand(1)),
getValue(I.getArgOperand(2)), Flags));
return;
+ case Intrinsic::vector_shuffle:
+ setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, sdl,
+ TLI.getValueType(DAG.getDataLayout(), I.getType()),
+ getValue(I.getArgOperand(0)),
+ getValue(I.getArgOperand(1)), Flags));
+ return;
case Intrinsic::experimental_convergence_anchor:
case Intrinsic::experimental_convergence_entry:
case Intrinsic::experimental_convergence_loop:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8bc1707b47b314..c9f6a04c624737 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -474,6 +474,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::MGATHER: return "masked_gather";
case ISD::MSCATTER: return "masked_scatter";
case ISD::VECTOR_COMPRESS: return "vector_compress";
+ case ISD::VECTOR_SHUFFLE_VAR: return "vector_shuffle_var";
case ISD::VAARG: return "vaarg";
case ISD::VACOPY: return "vacopy";
case ISD::VAEND: return "vaend";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7c97dfb2c806a1..c542644964c027 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13680,6 +13680,53 @@ SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
return DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo, Alignment);
}
+SDValue TargetLowering::expandVECTOR_SHUFFLE_VAR(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Node);
+ SDValue V = Node->getOperand(0);
+ SDValue Mask = Node->getOperand(1);
+
+ EVT VecVT = Node->getValueType(0);
+ EVT ScalarVT = VecVT.getScalarType();
+ EVT MaskScalarVT = Mask.getValueType().getScalarType();
+
+ // Needs to be handled by targets that have scalable vector types.
+ if (VecVT.isScalableVector())
+ report_fatal_error(
+ "Cannot expand vector_shuffle_var for scalable vectors.");
+
+ // Expand through memory:
+ // Alloca VecVT Ptr
+ // Store V, Ptr
+ // Res[i] = Load Ptr + umin(zext(Mask[i]), NumElts - 1) * sizeof(Elt)
+ // Clamping the index keeps out-of-range loads in-bounds; those lanes are
+ // poison, so any in-bounds element is a valid result.
+ Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
+ SDValue StackPtr = DAG.CreateStackTemporary(VecVT.getStoreSize(), Alignment);
+ auto &MF = DAG.getMachineFunction();
+ auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
+ auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
+
+ SDValue Chain =
+ DAG.getStore(DAG.getEntryNode(), DL, V, StackPtr, PtrInfo, Alignment);
+
+ // Freeze the whole mask once, rather than each extracted index, in case it
+ // has poison/undef elements.
+ Mask = DAG.getFreeze(Mask);
+
+ MVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
+ unsigned NumElts = VecVT.getVectorNumElements();
+ SmallVector<SDValue, 16> Elts;
+ Elts.reserve(NumElts);
+ for (unsigned I = 0; I < NumElts; ++I) {
+ SDValue Idx = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I);
+ Idx = DAG.getZExtOrTrunc(Idx, DL, IdxVT);
+ SDValue EltPtr = getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
+ Elts.push_back(DAG.getLoad(ScalarVT, DL, Chain, EltPtr, PtrInfo));
+ }
+ return DAG.getBuildVector(VecVT, DL, Elts);
+}
+
SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
SDLoc DL(Node);
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f852f7551ce16c..c689cebb77542c 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -951,7 +951,9 @@ void TargetLoweringBase::initActions() {
Expand);
// Only some target support these vector operations. Default them to Expand.
- setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
+ setOperationAction(
+ {ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH, ISD::VECTOR_SHUFFLE_VAR}, VT,
+ Expand);
// cttz.elts defaults to expand.
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1440b95896474f..9c0c37b4715c0b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -7057,6 +7057,17 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
}
break;
}
+ case Intrinsic::vector_shuffle: {
+ // The vector operand and the result share a type via LLVMMatchType and the
+ // mask is constrained to a vector of integers, so only the element count of
+ // the independently-overloaded mask type needs checking.
+ auto *RetTy = cast<VectorType>(Call.getType());
+ auto *MaskTy = cast<VectorType>(Call.getArgOperand(1)->getType());
+
+ Check(MaskTy->getElementCount() == RetTy->getElementCount(),
+ "Mask and return type must have the same number of elements.", &Call);
+ break;
+ }
case Intrinsic::vector_insert: {
Value *Vec = Call.getArgOperand(0);
Value *SubVec = Call.getArgOperand(1);
diff --git a/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll b/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
new file mode 100644
index 00000000000000..ed9adb2973e766
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-shuffle-var-expand.ll
@@ -0,0 +1,485 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown | FileCheck %s --check-prefixes=CHECK,LE
+; RUN: llc < %s -mtriple=aarch64_be-unknown | FileCheck %s --check-prefixes=CHECK,BE
+
+; AArch64 has no lowering for VECTOR_SHUFFLE_VAR yet, so the shuffle goes
+; through the generic stack expansion. Element 0 is at the lowest address in
+; memory on both endiannesses, so the big-endian expansion only differs in how
+; the vector is stored and the elements reloaded.
+
+define <4 x i32> @shuffle_v4i32(<4 x i32> %v, <4 x i32> %mask) {
+; LE-LABEL: shuffle_v4i32:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: mov x11, sp
+; LE-NEXT: bfi x8, x10, #2, #2
+; LE-NEXT: mov w10, v1.s[2]
+; LE-NEXT: bfi x11, x9, #2, #2
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: mov w8, v1.s[3]
+; LE-NEXT: ld1 { v0.s }[1], [x11]
+; LE-NEXT: bfi x9, x10, #2, #2
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x8, #2, #2
+; LE-NEXT: ld1 { v0.s }[2], [x9]
+; LE-NEXT: ld1 { v0.s }[3], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v4i32:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4s, v1.4s
+; BE-NEXT: mov x8, sp
+; BE-NEXT: mov x13, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov w9, v1.s[2]
+; BE-NEXT: mov w10, v1.s[1]
+; BE-NEXT: fmov w12, s1
+; BE-NEXT: mov w11, v1.s[3]
+; BE-NEXT: bfi x8, x12, #2, #2
+; BE-NEXT: bfi x13, x9, #2, #2
+; BE-NEXT: mov x9, sp
+; BE-NEXT: bfi x9, x10, #2, #2
+; BE-NEXT: mov x10, sp
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: bfi x10, x11, #2, #2
+; BE-NEXT: ldr s1, [x13]
+; BE-NEXT: ld1 { v0.s }[1], [x9]
+; BE-NEXT: ld1 { v1.s }[1], [x10]
+; BE-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; BE-NEXT: rev64 v0.4s, v0.4s
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> %v, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <16 x i8> @shuffle_v16i8(<16 x i8> %v, <16 x i8> %mask) {
+; LE-LABEL: shuffle_v16i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: umov w10, v1.b[0]
+; LE-NEXT: umov w11, v1.b[8]
+; LE-NEXT: mov x13, sp
+; LE-NEXT: umov w9, v1.b[1]
+; LE-NEXT: umov w12, v1.b[9]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: umov w15, v1.b[11]
+; LE-NEXT: bfxil x13, x10, #0, #4
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfxil x8, x9, #0, #4
+; LE-NEXT: umov w9, v1.b[2]
+; LE-NEXT: umov w11, v1.b[10]
+; LE-NEXT: ldr b0, [x13]
+; LE-NEXT: bfxil x10, x12, #0, #4
+; LE-NEXT: ldr b2, [x14]
+; LE-NEXT: mov x12, sp
+; LE-NEXT: umov w13, v1.b[3]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: ld1 { v0.b }[1], [x8]
+; LE-NEXT: ld1 { v2.b }[1], [x10]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: bfxil x12, x9, #0, #4
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: umov w9, v1.b[4]
+; LE-NEXT: mov x10, sp
+; LE-NEXT: umov w11, v1.b[12]
+; LE-NEXT: ld1 { v0.b }[2], [x12]
+; LE-NEXT: ld1 { v2.b }[2], [x14]
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: bfxil x10, x15, #0, #4
+; LE-NEXT: mov x12, sp
+; LE-NEXT: umov w13, v1.b[5]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: umov w15, v1.b[13]
+; LE-NEXT: bfxil x12, x9, #0, #4
+; LE-NEXT: ld1 { v0.b }[3], [x8]
+; LE-NEXT: ld1 { v2.b }[3], [x10]
+; LE-NEXT: bfxil x14, x11, #0, #4
+; LE-NEXT: mov x8, sp
+; LE-NEXT: umov w9, v1.b[6]
+; LE-NEXT: mov x10, sp
+; LE-NEXT: umov w11, v1.b[14]
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: umov w13, v1.b[15]
+; LE-NEXT: ld1 { v0.b }[4], [x12]
+; LE-NEXT: ld1 { v2.b }[4], [x14]
+; LE-NEXT: bfxil x10, x15, #0, #4
+; LE-NEXT: umov w12, v1.b[7]
+; LE-NEXT: mov x14, sp
+; LE-NEXT: mov x15, sp
+; LE-NEXT: bfxil x14, x9, #0, #4
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ld1 { v0.b }[5], [x8]
+; LE-NEXT: ld1 { v2.b }[5], [x10]
+; LE-NEXT: bfxil x15, x11, #0, #4
+; LE-NEXT: mov x8, sp
+; LE-NEXT: bfxil x9, x12, #0, #4
+; LE-NEXT: bfxil x8, x13, #0, #4
+; LE-NEXT: ld1 { v0.b }[6], [x14]
+; LE-NEXT: ld1 { v2.b }[6], [x15]
+; LE-NEXT: ld1 { v0.b }[7], [x9]
+; LE-NEXT: ld1 { v2.b }[7], [x8]
+; LE-NEXT: zip1 v0.2d, v0.2d, v2.2d
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v16i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.16b, v1.16b
+; BE-NEXT: mov x10, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: umov w8, v1.b[0]
+; BE-NEXT: umov w9, v1.b[1]
+; BE-NEXT: umov w11, v1.b[2]
+; BE-NEXT: bfxil x10, x8, #0, #4
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[3]
+; BE-NEXT: ldr b0, [x10]
+; BE-NEXT: umov w10, v1.b[4]
+; BE-NEXT: ld1 { v0.b }[1], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x11, #0, #4
+; BE-NEXT: ld1 { v0.b }[2], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[5]
+; BE-NEXT: ld1 { v0.b }[3], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[6]
+; BE-NEXT: ld1 { v0.b }[4], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[7]
+; BE-NEXT: ld1 { v0.b }[5], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[8]
+; BE-NEXT: ld1 { v0.b }[6], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[9]
+; BE-NEXT: ld1 { v0.b }[7], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[10]
+; BE-NEXT: ld1 { v0.b }[8], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[11]
+; BE-NEXT: ld1 { v0.b }[9], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[12]
+; BE-NEXT: ld1 { v0.b }[10], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[13]
+; BE-NEXT: ld1 { v0.b }[11], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: umov w10, v1.b[14]
+; BE-NEXT: ld1 { v0.b }[12], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x9, #0, #4
+; BE-NEXT: umov w9, v1.b[15]
+; BE-NEXT: ld1 { v0.b }[13], [x8]
+; BE-NEXT: mov x8, sp
+; BE-NEXT: bfxil x8, x10, #0, #4
+; BE-NEXT: mov x10, sp
+; BE-NEXT: bfxil x10, x9, #0, #4
+; BE-NEXT: ld1 { v0.b }[14], [x8]
+; BE-NEXT: ld1 { v0.b }[15], [x10]
+; BE-NEXT: rev64 v0.16b, v0.16b
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <16 x i8> @llvm.vector.shuffle.v16i8.v16i8(<16 x i8> %v, <16 x i8> %mask)
+ ret <16 x i8> %res
+}
+
+define <2 x i64> @shuffle_v2i64(<2 x i64> %v, <2 x i64> %mask) {
+; LE-LABEL: shuffle_v2i64:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov x10, d1
+; LE-NEXT: mov x9, v1.d[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: bfi x8, x10, #3, #1
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x9, #3, #1
+; LE-NEXT: ldr d0, [x8]
+; LE-NEXT: ld1 { v0.d }[1], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v2i64:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov x8, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: fmov x10, d1
+; BE-NEXT: mov x9, v1.d[1]
+; BE-NEXT: bfi x8, x10, #3, #1
+; BE-NEXT: mov x10, sp
+; BE-NEXT: bfi x10, x9, #3, #1
+; BE-NEXT: ldr d0, [x8]
+; BE-NEXT: ld1 { v0.d }[1], [x10]
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <2 x i64> @llvm.vector.shuffle.v2i64.v2i64(<2 x i64> %v, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+
+define <4 x float> @shuffle_v4f32(<4 x float> %v, <4 x i32> %mask) {
+; LE-LABEL: shuffle_v4f32:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: mov x8, sp
+; LE-NEXT: str q0, [sp]
+; LE-NEXT: mov x11, sp
+; LE-NEXT: bfi x8, x10, #2, #2
+; LE-NEXT: mov w10, v1.s[2]
+; LE-NEXT: bfi x11, x9, #2, #2
+; LE-NEXT: mov x9, sp
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: mov w8, v1.s[3]
+; LE-NEXT: ld1 { v0.s }[1], [x11]
+; LE-NEXT: bfi x9, x10, #2, #2
+; LE-NEXT: mov x10, sp
+; LE-NEXT: bfi x10, x8, #2, #2
+; LE-NEXT: ld1 { v0.s }[2], [x9]
+; LE-NEXT: ld1 { v0.s }[3], [x10]
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: shuffle_v4f32:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4s, v1.4s
+; BE-NEXT: mov x8, sp
+; BE-NEXT: mov x13, sp
+; BE-NEXT: str q0, [sp]
+; BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; BE-NEXT: mov w9, v1.s[2]
+; BE-NEXT: mov w10, v1.s[1]
+; BE-NEXT: fmov w12, s1
+; BE-NEXT: mov w11, v1.s[3]
+; BE-NEXT: bfi x8, x12, #2, #2
+; BE-NEXT: bfi x13, x9, #2, #2
+; BE-NEXT: mov x9, sp
+; BE-NEXT: bfi x9, x10, #2, #2
+; BE-NEXT: mov x10, sp
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: bfi x10, x11, #2, #2
+; BE-NEXT: ldr s1, [x13]
+; BE-NEXT: ld1 { v0.s }[1], [x9]
+; BE-NEXT: ld1 { v1.s }[1], [x10]
+; BE-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; BE-NEXT: rev64 v0.4s, v0.4s
+; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x float> @llvm.vector.shuffle.v4f32.v4i32(<4 x float> %v, <4 x i32> %mask)
+ ret <4 x float> %res
+}
+
+; The v4i8 mask is promoted to v4i16 with a zero extension: indices are
+; unsigned.
+define <4 x i16> @promote_mask_v4i16_v4i8(<4 x i16> %v, <4 x i8> %mask) {
+; LE-LABEL: promote_mask_v4i16_v4i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bic v1.4h, #255, lsl #8
+; LE-NEXT: umov w8, v1.h[0]
+; LE-NEXT: umov w9, v1.h[1]
+; LE-NEXT: umov w11, v1.h[2]
+; LE-NEXT: bfi x10, x8, #1, #2
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x9, #1, #2
+; LE-NEXT: umov w9, v1.h[3]
+; LE-NEXT: ldr h0, [x10]
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: ld1 { v0.h }[1], [x8]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x11, #1, #2
+; LE-NEXT: bfi x10, x9, #1, #2
+; LE-NEXT: ld1 { v0.h }[2], [x8]
+; LE-NEXT: ld1 { v0.h }[3], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_mask_v4i16_v4i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4h, v1.4h
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: bic v1.4h, #255, lsl #8
+; BE-NEXT: umov w8, v1.h[0]
+; BE-NEXT: umov w9, v1.h[1]
+; BE-NEXT: umov w11, v1.h[2]
+; BE-NEXT: bfi x10, x8, #1, #2
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x9, #1, #2
+; BE-NEXT: umov w9, v1.h[3]
+; BE-NEXT: ldr h0, [x10]
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: ld1 { v0.h }[1], [x8]
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x11, #1, #2
+; BE-NEXT: bfi x10, x9, #1, #2
+; BE-NEXT: ld1 { v0.h }[2], [x8]
+; BE-NEXT: ld1 { v0.h }[3], [x10]
+; BE-NEXT: rev64 v0.4h, v0.4h
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i16> @llvm.vector.shuffle.v4i16.v4i8(<4 x i16> %v, <4 x i8> %mask)
+ ret <4 x i16> %res
+}
+
+; Both the result (and so the source) and the mask are promoted.
+define <4 x i8> @promote_res_v4i8(<4 x i8> %v, <4 x i8> %mask) {
+; LE-LABEL: promote_res_v4i8:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bic v1.4h, #255, lsl #8
+; LE-NEXT: umov w8, v1.h[0]
+; LE-NEXT: umov w9, v1.h[1]
+; LE-NEXT: umov w11, v1.h[2]
+; LE-NEXT: bfi x10, x8, #1, #2
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x9, #1, #2
+; LE-NEXT: umov w9, v1.h[3]
+; LE-NEXT: ldr h0, [x10]
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: ld1 { v0.h }[1], [x8]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: bfi x8, x11, #1, #2
+; LE-NEXT: bfi x10, x9, #1, #2
+; LE-NEXT: ld1 { v0.h }[2], [x8]
+; LE-NEXT: ld1 { v0.h }[3], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_res_v4i8:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.4h, v1.4h
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: bic v1.4h, #255, lsl #8
+; BE-NEXT: umov w8, v1.h[0]
+; BE-NEXT: umov w9, v1.h[1]
+; BE-NEXT: umov w11, v1.h[2]
+; BE-NEXT: bfi x10, x8, #1, #2
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x9, #1, #2
+; BE-NEXT: umov w9, v1.h[3]
+; BE-NEXT: ldr h0, [x10]
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: ld1 { v0.h }[1], [x8]
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: bfi x8, x11, #1, #2
+; BE-NEXT: bfi x10, x9, #1, #2
+; BE-NEXT: ld1 { v0.h }[2], [x8]
+; BE-NEXT: ld1 { v0.h }[3], [x10]
+; BE-NEXT: rev64 v0.4h, v0.4h
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <4 x i8> @llvm.vector.shuffle.v4i8.v4i8(<4 x i8> %v, <4 x i8> %mask)
+ ret <4 x i8> %res
+}
+
+define <2 x i16> @promote_res_v2i16(<2 x i16> %v, <2 x i16> %mask) {
+; LE-LABEL: promote_res_v2i16:
+; LE: // %bb.0:
+; LE-NEXT: sub sp, sp, #16
+; LE-NEXT: .cfi_def_cfa_offset 16
+; LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; LE-NEXT: fmov w10, s1
+; LE-NEXT: mov w9, v1.s[1]
+; LE-NEXT: add x8, sp, #8
+; LE-NEXT: str d0, [sp, #8]
+; LE-NEXT: bfi x8, x10, #2, #1
+; LE-NEXT: add x10, sp, #8
+; LE-NEXT: bfi x10, x9, #2, #1
+; LE-NEXT: ldr s0, [x8]
+; LE-NEXT: ld1 { v0.s }[1], [x10]
+; LE-NEXT: // kill: def $d0 killed $d0 killed $q0
+; LE-NEXT: add sp, sp, #16
+; LE-NEXT: ret
+;
+; BE-LABEL: promote_res_v2i16:
+; BE: // %bb.0:
+; BE-NEXT: sub sp, sp, #16
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: rev64 v1.2s, v1.2s
+; BE-NEXT: add x8, sp, #8
+; BE-NEXT: str d0, [sp, #8]
+; BE-NEXT: fmov w10, s1
+; BE-NEXT: mov w9, v1.s[1]
+; BE-NEXT: bfi x8, x10, #2, #1
+; BE-NEXT: add x10, sp, #8
+; BE-NEXT: bfi x10, x9, #2, #1
+; BE-NEXT: ldr s0, [x8]
+; BE-NEXT: ld1 { v0.s }[1], [x10]
+; BE-NEXT: rev64 v0.2s, v0.2s
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+ %res = call <2 x i16> @llvm.vector.shuffle.v2i16.v2i16(<2 x i16> %v, <2 x i16> %mask)
+ ret <2 x i16> %res
+}
+
+; v1i64 is legal on AArch64, so this is a one-element expansion rather than a
+; scalarization.
+define <1 x i64> @shuffle_v1i64(<1 x i64> %v, <1 x i64> %mask) {
+; CHECK-LABEL: shuffle_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: ret
+ %res = call <1 x i64> @llvm.vector.shuffle.v1i64.v1i64(<1 x i64> %v, <1 x i64> %mask)
+ ret <1 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll b/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
new file mode 100644
index 00000000000000..fc58dde431fc53
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-shuffle-var-expand.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; X86 has no lowering for VECTOR_SHUFFLE_VAR yet, so every subtarget goes
+; through the generic expansion: store the source to the stack and load each
+; result element through a clamped variable index.
+
+define <4 x i32> @shuffle_v4i32(<4 x i32> %v, <4 x i32> %mask) {
+; SSE2-LABEL: shuffle_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> %v, <4 x i32> %mask)
+ ret <4 x i32> %res
+}
+
+define <2 x i64> @shuffle_v2i64(<2 x i64> %v, <2 x i64> %mask) {
+; SSE2-LABEL: shuffle_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: andl $1, %eax
+; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: andl $1, %eax
+; SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX2-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT: vmovq %xmm1, %rax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
+; AVX512-NEXT: vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: retq
+ %res = call <2 x i64> @llvm.vector.shuffle.v2i64.v2i64(<2 x i64> %v, <2 x i64> %mask)
+ ret <2 x i64> %res
+}
+; Both operands are constant. The shuffle is not elementwise, so it must not be
+; constant folded lane by lane.
+define <4 x i32> @shuffle_v4i32_const() {
+; SSE2-LABEL: shuffle_v4i32_const:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps {{.*#+}} xmm0 = [40,30,20,10]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: shuffle_v4i32_const:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps {{.*#+}} xmm0 = [40,30,20,10]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shuffle_v4i32_const:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm0 = [40,30,20,10]
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i32(<4 x i32> <i32 10, i32 20, i32 30, i32 40>, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+ ret <4 x i32> %res
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll b/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
new file mode 100644
index 00000000000000..1548882977ae9d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-shuffle-var-legalize.ll
@@ -0,0 +1,1048 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; Type legalization of VECTOR_SHUFFLE_VAR. Every case ends in the generic stack
+; expansion since X86 has no lowering yet; the interesting part is how illegal
+; result and mask types are brought to legal ones first.
+
+; A one-element source: the only in-range index is 0, so the result is the
+; source element.
+define <1 x i32> @scalarize_v1i32(<1 x i32> %v, <1 x i32> %mask) {
+; CHECK-LABEL: scalarize_v1i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: retq
+ %res = call <1 x i32> @llvm.vector.shuffle.v1i32.v1i32(<1 x i32> %v, <1 x i32> %mask)
+ ret <1 x i32> %res
+}
+
+; The result is widened to v4i32 and the mask with it.
+define <3 x i32> @widen_v3i32(<3 x i32> %v, <3 x i32> %mask) {
+; SSE2-LABEL: widen_v3i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: widen_v3i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: widen_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <3 x i32> @llvm.vector.shuffle.v3i32.v3i32(<3 x i32> %v, <3 x i32> %mask)
+ ret <3 x i32> %res
+}
+
+; The result is widened to v4i32; the mask is widened and then re-typed to the
+; legal v4i32 index type.
+define <2 x i32> @widen_v2i32_v2i8(<2 x i32> %v, <2 x i8> %mask) {
+; SSE2-LABEL: widen_v2i32_v2i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: pextrw $0, %xmm1, %eax
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: widen_v2i32_v2i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrb $1, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $2, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $3, %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: widen_v2i32_v2i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrb $1, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $2, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrb $3, %xmm1, %eax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %res = call <2 x i32> @llvm.vector.shuffle.v2i32.v2i8(<2 x i32> %v, <2 x i8> %mask)
+ ret <2 x i32> %res
+}
+
+; Split when 256-bit vectors are illegal: each result half shuffles both source halves with the same
+; indices and selects per lane on which half the index falls in.
+define <8 x i32> @split_v8i32(<8 x i32> %v, <8 x i32> %mask) {
+; SSE2-LABEL: split_v8i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [4,4,4,4]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[3,3,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,1,1,1]
+; SSE2-NEXT: psubd %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm4, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm8, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm9[0]
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT: pxor %xmm4, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483652,2147483652,2147483652,2147483652]
+; SSE2-NEXT: movdqa %xmm2, %xmm9
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm9
+; SSE2-NEXT: movd %xmm5, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm6, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; SSE2-NEXT: pand %xmm9, %xmm0
+; SSE2-NEXT: pandn %xmm8, %xmm9
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: pxor %xmm3, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[3,3,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
+; SSE2-NEXT: movd %xmm8, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm8 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm9 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm5, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm6, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm9 = xmm9[0],xmm8[0]
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; SSE2-NEXT: pand %xmm2, %xmm1
+; SSE2-NEXT: pandn %xmm9, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm0, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm0, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i32(<8 x i32> %v, <8 x i32> %mask)
+ ret <8 x i32> %res
+}
+
+; Split with a mask whose element type is illegal on its own: the half masks are
+; re-typed to the legal v4i32 index type.
+define <8 x i32> @split_v8i32_v8i8(<8 x i32> %v, <8 x i8> %mask) {
+; SSE2-LABEL: split_v8i32_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [4,4,4,4,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pmaxub %xmm3, %xmm5
+; SSE2-NEXT: pcmpeqb %xmm2, %xmm5
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm2, %eax
+; SSE2-NEXT: movdqa %xmm2, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; SSE2-NEXT: movd %xmm7, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; SSE2-NEXT: movd %xmm6, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm7[0]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: psubb %xmm3, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; SSE2-NEXT: movd %xmm7, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
+; SSE2-NEXT: movd %xmm6, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm7[0]
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: movdqa %xmm4, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm2, %eax
+; SSE2-NEXT: movdqa %xmm2, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[3,3,3,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
+; SSE2-NEXT: movd %xmm7, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa %xmm4, %xmm8
+; SSE2-NEXT: psubb %xmm3, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT: pextrw $0, %xmm4, %eax
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm4[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %ecx
+; SSE2-NEXT: andl $3, %ecx
+; SSE2-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [4,4,4,4,4,4,4,4,u,u,u,u,u,u,u,u]
+; SSE2-NEXT: pmaxub %xmm8, %xmm5
+; SSE2-NEXT: pcmpeqb %xmm8, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: pand %xmm5, %xmm1
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: por %xmm5, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vpextrd $1, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vmovd %xmm0, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $2, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $3, %xmm0, %eax
+; AVX2-NEXT: andl $7, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpextrd $1, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm1, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vpextrd $1, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vmovd %xmm0, %ecx
+; AVX512-NEXT: andl $7, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $2, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrd $3, %xmm0, %eax
+; AVX512-NEXT: andl $7, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i8(<8 x i32> %v, <8 x i8> %mask)
+ ret <8 x i32> %res
+}
+
+; An i2 index can only reach elements 0-3, so after the split the high source
+; half is unreachable and each result half is a shuffle of the low half alone.
+define <8 x i32> @split_v8i32_v8i2(<8 x i32> %v, <8 x i2> %mask) {
+; SSE2-LABEL: split_v8i32_v8i2:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [3,3,3,3]
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i2:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i2(<8 x i32> %v, <8 x i2> %mask)
+ ret <8 x i32> %res
+}
+
+; Same with an i1 index, which can only reach elements 0 and 1.
+define <8 x i32> @split_v8i32_v8i1(<8 x i32> %v, <8 x i1> %mask) {
+; SSE2-LABEL: split_v8i32_v8i1:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,1,1,1]
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %xmm4, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: split_v8i32_v8i1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: .cfi_def_cfa_offset 16
+; AVX2-NEXT: .cfi_offset %rbp, -16
+; AVX2-NEXT: movq %rsp, %rbp
+; AVX2-NEXT: .cfi_def_cfa_register %rbp
+; AVX2-NEXT: andq $-32, %rsp
+; AVX2-NEXT: subq $64, %rsp
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vmovaps %ymm0, (%rsp)
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: vpextrw $4, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: vpextrw $0, %xmm1, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: movq %rbp, %rsp
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: .cfi_def_cfa %rsp, 8
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: split_v8i32_v8i1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: .cfi_def_cfa_offset 16
+; AVX512-NEXT: .cfi_offset %rbp, -16
+; AVX512-NEXT: movq %rsp, %rbp
+; AVX512-NEXT: .cfi_def_cfa_register %rbp
+; AVX512-NEXT: andq $-32, %rsp
+; AVX512-NEXT: subq $64, %rsp
+; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1
+; AVX512-NEXT: vpmovw2m %xmm1, %k3
+; AVX512-NEXT: kshiftrw $3, %k3, %k0
+; AVX512-NEXT: kshiftrw $2, %k3, %k1
+; AVX512-NEXT: kshiftrw $1, %k3, %k2
+; AVX512-NEXT: kshiftrw $7, %k3, %k4
+; AVX512-NEXT: kshiftrw $6, %k3, %k5
+; AVX512-NEXT: kshiftrw $4, %k3, %k6
+; AVX512-NEXT: kshiftrw $5, %k3, %k7
+; AVX512-NEXT: vmovaps %ymm0, (%rsp)
+; AVX512-NEXT: kmovd %k7, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: kmovd %k6, %ecx
+; AVX512-NEXT: andl $1, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k5, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k4, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: kmovd %k3, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512-NEXT: kmovd %k2, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $1, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: kmovd %k1, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $2, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: andl $1, %eax
+; AVX512-NEXT: vpinsrd $3, (%rsp,%rax,4), %xmm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: movq %rbp, %rsp
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: .cfi_def_cfa %rsp, 8
+; AVX512-NEXT: retq
+ %res = call <8 x i32> @llvm.vector.shuffle.v8i32.v8i1(<8 x i32> %v, <8 x i1> %mask)
+ ret <8 x i32> %res
+}
+
+; The result is legal but the mask is too wide and would be split: it is
+; truncated to the legal v4i32 index type instead. Out-of-range indices are
+; poison, so the truncation is a refinement.
+define <4 x i32> @mask_split_v4i32_v4i64(<4 x i32> %v, <4 x i64> %mask) {
+; SSE2-LABEL: mask_split_v4i32_v4i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: andl $3, %eax
+; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: mask_split_v4i32_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vmovq %xmm1, %rcx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: mask_split_v4i32_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vmovq %xmm1, %rcx
+; AVX512-NEXT: andl $3, %ecx
+; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512-NEXT: vpinsrd $1, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vmovq %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $2, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512-NEXT: andl $3, %eax
+; AVX512-NEXT: vpinsrd $3, -24(%rsp,%rax,4), %xmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v4i64(<4 x i32> %v, <4 x i64> %mask)
+ ret <4 x i32> %res
+}
+
+define <16 x i8> @mask_split_v16i8_v16i16(<16 x i8> %v, <16 x i16> %mask) {
+; SSE2-LABEL: mask_split_v16i8_v16i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm3, %xmm1
+; SSE2-NEXT: packuswb %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm4
+; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: andl $15, %eax
+; SSE2-NEXT: movzbl -24(%rsp,%rax), %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: mask_split_v16i8_v16i16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: movzbl -24(%rsp,%rcx), %ecx
+; AVX2-NEXT: vmovd %ecx, %xmm0
+; AVX2-NEXT: vpinsrb $1, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $2, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $2, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $3, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $3, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $4, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $4, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $5, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $5, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $6, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $6, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX2-NEXT: vpextrw $7, %xmm1, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $7, -24(%rsp,%rax), %xmm0, %xmm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $8, -24(%rsp,%rax), %xmm2, %xmm1
+; AVX2-NEXT: vpextrw $1, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $9, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $2, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $10, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $3, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $11, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $4, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $12, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $5, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $13, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $6, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $14, -24(%rsp,%rax), %xmm1, %xmm1
+; AVX2-NEXT: vpextrw $7, %xmm0, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpinsrb $15, -24(%rsp,%rax), %xmm1, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: mask_split_v16i8_v16i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vmovd %xmm1, %ecx
+; AVX512-NEXT: andl $15, %ecx
+; AVX512-NEXT: movzbl -24(%rsp,%rcx), %ecx
+; AVX512-NEXT: vmovd %ecx, %xmm0
+; AVX512-NEXT: vpinsrb $1, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $2, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $3, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $4, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $4, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $5, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $6, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $7, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vmovd %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $8, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $1, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $9, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $10, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $3, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $11, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $4, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $12, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $13, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $6, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $14, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vpextrw $7, %xmm1, %eax
+; AVX512-NEXT: andl $15, %eax
+; AVX512-NEXT: vpinsrb $15, -24(%rsp,%rax), %xmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %res = call <16 x i8> @llvm.vector.shuffle.v16i8.v16i16(<16 x i8> %v, <16 x i16> %mask)
+ ret <16 x i8> %res
+}
diff --git a/llvm/test/Verifier/vector-shuffle-var.ll b/llvm/test/Verifier/vector-shuffle-var.ll
new file mode 100644
index 00000000000000..5bb5748cb4499e
--- /dev/null
+++ b/llvm/test/Verifier/vector-shuffle-var.ll
@@ -0,0 +1,20 @@
+; RUN: not opt -passes=verify -S < %s 2>&1 | FileCheck %s
+
+; CHECK: Mask and return type must have the same number of elements.
+define <4 x i32> @mask_too_long(<4 x i32> %v, <8 x i8> %mask) {
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v8i8(<4 x i32> %v, <8 x i8> %mask)
+ ret <4 x i32> %res
+}
+
+; CHECK: Mask and return type must have the same number of elements.
+define <4 x i32> @mask_too_short(<4 x i32> %v, <2 x i8> %mask) {
+ %res = call <4 x i32> @llvm.vector.shuffle.v4i32.v2i8(<4 x i32> %v, <2 x i8> %mask)
+ ret <4 x i32> %res
+}
+
+; A fixed-length mask never has the same element count as a scalable result.
+; CHECK: Mask and return type must have the same number of elements.
+define <vscale x 4 x i32> @mixed_scalable(<vscale x 4 x i32> %v, <4 x i8> %mask) {
+ %res = call <vscale x 4 x i32> @llvm.vector.shuffle.nxv4i32.v4i8(<vscale x 4 x i32> %v, <4 x i8> %mask)
+ ret <vscale x 4 x i32> %res
+}
More information about the llvm-commits
mailing list