[llvm] [X86] Add combineTruncateOfConcat and generalise EltsFromConsecutiveLoads strided emission (PR #208869)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 10 20:50:43 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/208869
>From a864c504497559e2682d8970887565e0557c29b9 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 11 Jul 2026 03:09:43 +0000
Subject: [PATCH] [X86] Add combineTruncateOfConcat, emit generic strided-load
truncation
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 126 +++++++++----
llvm/test/CodeGen/X86/avx10_2bf16-fma.ll | 177 ++++++------------
llvm/test/CodeGen/X86/avx512-calling-conv.ll | 76 ++++----
.../CodeGen/X86/avx512-masked_memop-16-8.ll | 64 +++----
llvm/test/CodeGen/X86/build-vector-128.ll | 54 ++++--
llvm/test/CodeGen/X86/build-vector-256.ll | 68 +++----
.../CodeGen/X86/buildvec-strided-loads.ll | 119 +++++-------
7 files changed, 326 insertions(+), 358 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 64d298d26b84f..889f17912199f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8091,29 +8091,23 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
if (WideEltBits != 0) {
MVT WideEltVT = MVT::getIntegerVT(WideEltBits);
MVT SrcEltVT = MVT::getIntegerVT(BaseSizeInBits);
- // VTRUNC writes the truncated values to the low lanes of an xmm with
- // zero padding above.
- MVT TruncDstVT = MVT::getVectorVT(SrcEltVT, 128 / BaseSizeInBits);
- unsigned TruncDstLanes = TruncDstVT.getVectorNumElements();
- MVT ConcatVT = MVT::getVectorVT(SrcEltVT, NumElems);
- if (NumElems > TruncDstLanes && !TLI.isTypeLegal(ConcatVT))
- return SDValue();
- // Try wider register sizes first.
+ // Each piece truncates to its narrowest legal vector. The truncate
+ // combines and lowering pick the final narrowing from there.
for (unsigned WideRegBits : {512u, 256u, 128u}) {
unsigned LanesPerWideLoad = WideRegBits / WideEltBits;
if (LanesPerWideLoad < 2 || NumElems % LanesPerWideLoad != 0)
continue;
- if (LanesPerWideLoad > TruncDstLanes)
- continue; // VTRUNC dest must hold all good lanes of one piece.
+ unsigned PieceEltBits =
+ std::max(BaseSizeInBits, 128 / LanesPerWideLoad);
+ MVT PieceEltVT = MVT::getIntegerVT(PieceEltBits);
MVT WideVT = MVT::getVectorVT(WideEltVT, LanesPerWideLoad);
- if (!TLI.isTypeLegal(WideVT) || !TLI.isTypeLegal(TruncDstVT))
+ MVT PieceVT = MVT::getVectorVT(PieceEltVT, LanesPerWideLoad);
+ MVT ConcatVT = MVT::getVectorVT(PieceEltVT, NumElems);
+ MVT TruncVT = MVT::getVectorVT(SrcEltVT, NumElems);
+ if (!TLI.isTypeLegal(WideVT) || !TLI.isTypeLegal(PieceVT) ||
+ !TLI.isTypeLegal(ConcatVT) || !TLI.isTypeLegal(TruncVT))
continue;
unsigned NumWideLoads = NumElems / LanesPerWideLoad;
- // VTRUNC has no non-AVX-512 lowering so the i16 to i8 form needs BWI.
- bool Partial = LanesPerWideLoad != TruncDstLanes;
- if (Partial && (!Subtarget.hasAVX512() ||
- (WideEltBits == 16 && !Subtarget.hasBWI())))
- continue;
unsigned BytesPerWideLoad = WideRegBits / 8;
auto MMOFlags = LDBase->getMemOperand()->getFlags();
SDValue BasePtr = LDBase->getBasePtr();
@@ -8130,32 +8124,14 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
for (auto *LD : Loads)
if (LD)
DAG.makeEquivalentMemoryOrdering(LD, Ld);
- unsigned TruncOp = Partial ? X86ISD::VTRUNC : ISD::TRUNCATE;
- Pieces.push_back(DAG.getNode(TruncOp, DL, TruncDstVT, Ld));
- }
- // Pairwise shuffle the low halves until each piece is full.
- if (Partial) {
- unsigned GoodLanes = LanesPerWideLoad;
- while (Pieces.size() > 1 && GoodLanes < TruncDstLanes) {
- assert((TruncDstLanes % GoodLanes) == 0 &&
- "Illegal VTRUNC packing");
- SmallVector<SDValue, 8> Next;
- SmallVector<int, 16> Mask(TruncDstLanes, -1);
- for (unsigned I = 0; I != GoodLanes; ++I) {
- Mask[I] = I;
- Mask[GoodLanes + I] = TruncDstLanes + I;
- }
- for (unsigned J = 0, E = Pieces.size() - 1; J < E; J += 2)
- Next.push_back(DAG.getVectorShuffle(TruncDstVT, DL, Pieces[J],
- Pieces[J + 1], Mask));
- Pieces = std::move(Next);
- GoodLanes *= 2;
- }
+ Pieces.push_back(DAG.getNode(ISD::TRUNCATE, DL, PieceVT, Ld));
}
- if (Pieces.size() == 1)
- return DAG.getBitcast(VT, Pieces[0]);
-
- SDValue Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Pieces);
+ SDValue Result = Pieces[0];
+ if (Pieces.size() > 1)
+ Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Pieces);
+ // Pieces may have stopped wider than the source width. Finish here.
+ if (PieceEltBits != BaseSizeInBits)
+ Result = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Result);
return DAG.getBitcast(VT, Result);
}
}
@@ -55868,6 +55844,70 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
PMADDBuilder);
}
+// Fold trunc(concat(trunc(x0), trunc(x1), ...)) -> shuffle(vtrunc(x0), ...).
+// A truncate of the concat narrows twice. VTRUNC narrows each source straight
+// to its final sub-128-bit width leaving only the shuffles.
+static SDValue combineTruncateOfConcat(SDValue Src, EVT VT, const SDLoc &DL,
+ SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ if (!Subtarget.hasAVX512() || !VT.isVector() || !Src.hasOneUse())
+ return SDValue();
+
+ SmallVector<SDValue, 4> Ops;
+ if (!collectConcatOps(Src.getNode(), Ops, DAG) || Ops.size() < 2 ||
+ !isPowerOf2_32(Ops.size()))
+ return SDValue();
+
+ // Every piece must be a truncate from one common source type.
+ using namespace SDPatternMatch;
+ EVT InVT = Ops[0].getOperand(0).getValueType();
+ if (!all_of(Ops, [InVT](SDValue Op) {
+ return sd_match(Op, m_OneUse(m_Trunc(m_SpecificVT(InVT))));
+ }))
+ return SDValue();
+
+ // VTRUNC writes to the low lanes of an xmm, so it only helps when a piece is
+ // narrower than that. It needs a 512-bit source (or 256 with VLX), and BWI
+ // for the i16 to i8 form.
+ unsigned InBits = InVT.getSizeInBits();
+ unsigned PieceLanes = InVT.getVectorNumElements();
+ MVT DstEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
+ unsigned TruncDstLanes = 128 / DstEltVT.getSizeInBits();
+ if (PieceLanes >= TruncDstLanes ||
+ (InBits != 512 && !(InBits == 256 && Subtarget.hasVLX())) ||
+ (InVT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI()))
+ return SDValue();
+
+ MVT TruncDstVT = MVT::getVectorVT(DstEltVT, TruncDstLanes);
+ SmallVector<SDValue, 8> Pieces;
+ for (SDValue Op : Ops)
+ Pieces.push_back(
+ DAG.getNode(X86ISD::VTRUNC, DL, TruncDstVT, Op.getOperand(0)));
+
+ // Pairwise shuffle the low halves until each piece is full.
+ unsigned GoodLanes = PieceLanes;
+ while (Pieces.size() > 1 && GoodLanes < TruncDstLanes) {
+ assert((TruncDstLanes % GoodLanes) == 0 && "Illegal VTRUNC packing");
+ SmallVector<SDValue, 8> Next;
+ SmallVector<int, 16> Mask(TruncDstLanes, -1);
+ for (unsigned I = 0; I != GoodLanes; ++I) {
+ Mask[I] = I;
+ Mask[GoodLanes + I] = TruncDstLanes + I;
+ }
+ for (unsigned J = 0, E = Pieces.size() - 1; J < E; J += 2)
+ Next.push_back(
+ DAG.getVectorShuffle(TruncDstVT, DL, Pieces[J], Pieces[J + 1], Mask));
+ Pieces = std::move(Next);
+ GoodLanes *= 2;
+ }
+ if (Pieces.size() == 1)
+ return DAG.getBitcast(VT, Pieces[0]);
+
+ MVT ConcatVT = MVT::getVectorVT(DstEltVT, Pieces.size() * TruncDstLanes);
+ return DAG.getBitcast(VT,
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Pieces));
+}
+
static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
EVT VT = N->getValueType(0);
@@ -55891,6 +55931,10 @@ static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
return V;
+ // Try to combine a truncate of concatenated truncates.
+ if (SDValue V = combineTruncateOfConcat(Src, VT, DL, DAG, Subtarget))
+ return V;
+
// Fold trunc(srl(load(p),amt)) -> load(p+amt/8)
// If we're shifting down byte aligned bit chunks from a larger load for
// truncation, see if we can convert the shift into a pointer offset instead.
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
index d79f0cc79b5f4..26cf3aba60407 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
@@ -647,155 +647,100 @@ define <29 x bfloat> @fuse_v19bf16_load(<29 x bfloat> %x, <29 x bfloat> %y, ptr
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm9
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1],xmm11[2],xmm13[2],xmm11[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1],xmm10[2],xmm13[2],xmm10[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],xmm11[0]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm9 = xmm9[0],xmm11[0]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm10, %ymm10
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm9, %ymm9
-; AVXNECONVERT-NEXT: vmulps %ymm10, %ymm9, %ymm9
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm9 = xmm9[0],xmm10[0],zero,zero
+; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm9 = xmm9[0,1],xmm10[0],xmm9[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm10 = xmm11[0],xmm10[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm11[0]
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm10, %ymm10
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3]
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm8 = xmm8[0],xmm10[0],zero,zero
+; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm8 = xmm8[0,1],xmm10[0],xmm8[3]
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm9, %ymm9
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm8, %ymm8
-; AVXNECONVERT-NEXT: vmulps %ymm10, %ymm8, %ymm8
+; AVXNECONVERT-NEXT: vmulps %ymm9, %ymm8, %ymm8
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm1
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],zero,zero
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],zero,zero
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm8, %xmm3
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm1
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vmulps %ymm0, %ymm2, %ymm0
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm4
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm4 = xmm4[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: vmulps %ymm2, %ymm3, %ymm2
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm2, %ymm1
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vaddps %ymm2, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm9, %xmm3
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm8, %xmm3
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm2
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
-; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm1, 56(%rdi)
-; AVXNECONVERT-NEXT: vmovq %xmm1, 48(%rdi)
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm1
-; AVXNECONVERT-NEXT: vmovaps %xmm1, 32(%rdi)
-; AVXNECONVERT-NEXT: vmovaps %ymm0, (%rdi)
+; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: vmovaps %xmm0, 32(%rdi)
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm0
+; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, 56(%rdi)
+; AVXNECONVERT-NEXT: vmovq %xmm0, 48(%rdi)
+; AVXNECONVERT-NEXT: vmovaps %ymm1, (%rdi)
; AVXNECONVERT-NEXT: vzeroupper
; AVXNECONVERT-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index 2e7425c13009f..d0bab28c2e98e 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -2429,29 +2429,29 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
; SKX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; SKX-NEXT: vptestmb %zmm2, %zmm0, %k0
-; SKX-NEXT: vmovdqu64 792(%rsp), %zmm0
-; SKX-NEXT: vpmovqb %zmm0, %xmm0
-; SKX-NEXT: vmovdqu64 920(%rsp), %zmm1
+; SKX-NEXT: vmovdqu64 728(%rsp), %zmm0
+; SKX-NEXT: vmovdqu64 792(%rsp), %zmm1
+; SKX-NEXT: vmovdqu64 856(%rsp), %zmm3
+; SKX-NEXT: vmovdqu64 920(%rsp), %zmm4
; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vmovdqu64 472(%rsp), %zmm3
-; SKX-NEXT: vmovdqu64 536(%rsp), %zmm4
-; SKX-NEXT: vmovdqu64 600(%rsp), %zmm5
-; SKX-NEXT: vmovdqu64 664(%rsp), %zmm6
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; SKX-NEXT: vpmovqb %zmm4, %xmm1
-; SKX-NEXT: vpmovqb %zmm6, %xmm4
-; SKX-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
-; SKX-NEXT: vmovdqu64 728(%rsp), %zmm4
; SKX-NEXT: vpmovqb %zmm4, %xmm4
-; SKX-NEXT: vmovdqu64 856(%rsp), %zmm6
-; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX-NEXT: vpmovqb %zmm6, %xmm1
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm4, %ymm1
+; SKX-NEXT: vmovdqu64 472(%rsp), %zmm5
+; SKX-NEXT: vmovdqu64 536(%rsp), %zmm6
+; SKX-NEXT: vmovdqu64 600(%rsp), %zmm7
+; SKX-NEXT: vmovdqu64 664(%rsp), %zmm8
+; SKX-NEXT: vpmovqb %zmm6, %xmm6
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
+; SKX-NEXT: vpmovqb %zmm8, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm4
+; SKX-NEXT: vpmovqb %zmm0, %xmm0
+; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm4, %zmm1
; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vpmovqb %zmm5, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; SKX-NEXT: vpmovqb %zmm5, %xmm3
+; SKX-NEXT: vpmovqb %zmm7, %xmm4
; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
-; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; SKX-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; SKX-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
; SKX-NEXT: vptestmb %zmm2, %zmm0, %k1
; SKX-NEXT: kmovq %k1, %rdx
; SKX-NEXT: kmovq %k0, %rax
@@ -2482,29 +2482,29 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
;
; FASTISEL-LABEL: PR179334:
; FASTISEL: ## %bb.0:
-; FASTISEL-NEXT: vmovdqu64 792(%rsp), %zmm0
-; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
-; FASTISEL-NEXT: vmovdqu64 920(%rsp), %zmm1
+; FASTISEL-NEXT: vmovdqu64 728(%rsp), %zmm0
+; FASTISEL-NEXT: vmovdqu64 792(%rsp), %zmm1
+; FASTISEL-NEXT: vmovdqu64 856(%rsp), %zmm2
+; FASTISEL-NEXT: vmovdqu64 920(%rsp), %zmm3
; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
-; FASTISEL-NEXT: vmovdqu64 472(%rsp), %zmm2
-; FASTISEL-NEXT: vmovdqu64 536(%rsp), %zmm3
-; FASTISEL-NEXT: vmovdqu64 600(%rsp), %zmm4
-; FASTISEL-NEXT: vmovdqu64 664(%rsp), %zmm5
; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vpmovqb %zmm5, %xmm5
-; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; FASTISEL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm1
-; FASTISEL-NEXT: vmovdqu64 728(%rsp), %zmm3
-; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; FASTISEL-NEXT: vmovdqu64 856(%rsp), %zmm1
-; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
-; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; FASTISEL-NEXT: vmovdqu64 472(%rsp), %zmm4
+; FASTISEL-NEXT: vmovdqu64 536(%rsp), %zmm5
+; FASTISEL-NEXT: vmovdqu64 600(%rsp), %zmm6
+; FASTISEL-NEXT: vmovdqu64 664(%rsp), %zmm7
+; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; FASTISEL-NEXT: vpmovqb %zmm5, %xmm3
+; FASTISEL-NEXT: vpmovqb %zmm7, %xmm5
+; FASTISEL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
; FASTISEL-NEXT: vpmovqb %zmm2, %xmm2
-; FASTISEL-NEXT: vpmovqb %zmm4, %xmm3
+; FASTISEL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FASTISEL-NEXT: vpmovqb %zmm4, %xmm2
+; FASTISEL-NEXT: vpmovqb %zmm6, %xmm3
; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
; FASTISEL-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k0
; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm0
diff --git a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
index b74771c825780..10c867a437397 100644
--- a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
@@ -220,52 +220,52 @@ define void @test_maskz_store_v192i8(ptr %p0, <192 x i1> %mask) nounwind {
; CHECK-NEXT: vpbroadcastb {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k1
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; CHECK-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
; CHECK-NEXT: vpmovqb %zmm2, %xmm2
; CHECK-NEXT: vpmovqb %zmm4, %xmm4
; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
-; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k2
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; CHECK-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm6
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm7
; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqb %zmm7, %xmm7
+; CHECK-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2
+; CHECK-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT: vpmovqb %zmm4, %xmm3
+; CHECK-NEXT: vpmovqb %zmm6, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
+; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
; CHECK-NEXT: vpmovqb %zmm2, %xmm2
; CHECK-NEXT: vpmovqb %zmm4, %xmm4
; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm6
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm7
+; CHECK-NEXT: vpmovqb %zmm5, %xmm5
+; CHECK-NEXT: vpmovqb %zmm7, %xmm7
+; CHECK-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2
+; CHECK-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; CHECK-NEXT: vpmovqb %zmm4, %xmm3
+; CHECK-NEXT: vpmovqb %zmm6, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k3
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovdqu8 %zmm0, 64(%rdi) {%k3}
diff --git a/llvm/test/CodeGen/X86/build-vector-128.ll b/llvm/test/CodeGen/X86/build-vector-128.ll
index e478e7037463a..f595178e1ed20 100644
--- a/llvm/test/CodeGen/X86/build-vector-128.ll
+++ b/llvm/test/CodeGen/X86/build-vector-128.ll
@@ -361,25 +361,25 @@ define <16 x i8> @test_buildvector_v16i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; SSE41-64-NEXT: pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
; SSE41-64-NEXT: retq
;
-; AVX-32-LABEL: test_buildvector_v16i8:
-; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: retl
+; AVX1-32-LABEL: test_buildvector_v16i8:
+; AVX1-32: # %bb.0:
+; AVX1-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v16i8:
; AVX-64: # %bb.0:
@@ -400,6 +400,22 @@ define <16 x i8> @test_buildvector_v16i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: retq
+;
+; AVX2-32-LABEL: test_buildvector_v16i8:
+; AVX2-32: # %bb.0:
+; AVX2-32-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm2
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm2, %ymm2
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-32-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; AVX2-32-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-32-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-32-NEXT: vzeroupper
+; AVX2-32-NEXT: retl
%ins0 = insertelement <16 x i8> undef, i8 %a0, i32 0
%ins1 = insertelement <16 x i8> %ins0, i8 %a1, i32 1
%ins2 = insertelement <16 x i8> %ins1, i8 %a2, i32 2
diff --git a/llvm/test/CodeGen/X86/build-vector-256.ll b/llvm/test/CodeGen/X86/build-vector-256.ll
index 2de346aee5857..b60e9f3dd3ee3 100644
--- a/llvm/test/CodeGen/X86/build-vector-256.ll
+++ b/llvm/test/CodeGen/X86/build-vector-256.ll
@@ -191,14 +191,13 @@ define <16 x i16> @test_buildvector_v16i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
; AVX2-64-NEXT: vpinsrw $5, %r9d, %xmm0, %xmm0
; AVX2-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX2-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-64-NEXT: vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm1
+; AVX2-64-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm2
+; AVX2-64-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],mem[0,2]
+; AVX2-64-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVX2-64-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-64-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-64-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX2-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-64-NEXT: retq
%ins0 = insertelement <16 x i16> undef, i16 %a0, i32 0
@@ -297,39 +296,26 @@ define <32 x i8> @test_buildvector_v32i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
;
; AVX2-32-LABEL: test_buildvector_v32i8:
; AVX2-32: # %bb.0:
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm2
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm2, %ymm2
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm1, %ymm1
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-32-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-32-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm3
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm4
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm4, %ymm4
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-32-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-32-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-32-NEXT: retl
;
; AVX2-64-LABEL: test_buildvector_v32i8:
diff --git a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
index e7f848a6e9ee9..670aa5e07b54c 100644
--- a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
+++ b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
@@ -88,23 +88,18 @@ define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v16i8_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzbl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu (%rdi), %ymm0
+; AVX2-NEXT: vmovdqu 32(%rdi), %ymm1
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: buildvec_v16i8_stride4:
@@ -211,25 +206,22 @@ define <8 x i16> @buildvec_v8i16_stride4(ptr %p) {
define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
; AVX2-LABEL: buildvec_v16i16_stride8:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzwl 64(%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrw $1, 72(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $2, 80(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $3, 88(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $4, 96(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $5, 104(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $6, 112(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $7, 120(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: movzwl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpinsrw $1, 8(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $2, 16(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $3, 24(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $4, 32(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $5, 40(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $6, 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $7, 56(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: vmovups (%rdi), %xmm0
+; AVX2-NEXT: vmovups 32(%rdi), %xmm1
+; AVX2-NEXT: vmovups 64(%rdi), %xmm2
+; AVX2-NEXT: vmovups 96(%rdi), %xmm3
+; AVX2-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVX2-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],mem[0,2]
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: buildvec_v16i16_stride8:
@@ -294,41 +286,26 @@ define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
define <32 x i8> @buildvec_v32i8_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v32i8_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzbl 64(%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrb $1, 68(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $2, 72(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $3, 76(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $4, 80(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, 84(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, 88(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, 92(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, 96(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, 100(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, 104(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, 108(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, 112(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, 116(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, 120(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, 124(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: movzbl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: vmovdqu (%rdi), %ymm0
+; AVX2-NEXT: vmovdqu 32(%rdi), %ymm1
+; AVX2-NEXT: vmovdqu 64(%rdi), %ymm2
+; AVX2-NEXT: vmovdqu 96(%rdi), %ymm3
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: buildvec_v32i8_stride4:
More information about the llvm-commits
mailing list