[llvm] Reapply "[X86] EltsFromConsecutiveLoads - handle trunc(wideload()) patterns" (#199371) (PR #208999)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 21:03:40 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/208999
>From d2ebe70dec34cbbdaf974e0c00f00a3d34e97251 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 12 Jul 2026 08:00:37 +0000
Subject: [PATCH 1/3] Reapply "[X86] EltsFromConsecutiveLoads - handle
trunc(wideload()) patterns" (#199371)
This reverts commit e9e21f520ccd8f4c292686dfdeb04eae080110e3, restoring the
original patch unchanged. The dereferenceability fix follows in the next
commit.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 96 ++
llvm/test/CodeGen/X86/avx512-calling-conv.ll | 915 ++++---------
llvm/test/CodeGen/X86/avx512-ext.ll | 126 +-
.../test/CodeGen/X86/avx512-insert-extract.ll | 286 ++---
llvm/test/CodeGen/X86/avx512-load-store.ll | 209 +--
llvm/test/CodeGen/X86/avx512-mask-op.ll | 126 +-
.../CodeGen/X86/avx512-masked_memop-16-8.ll | 267 ++--
llvm/test/CodeGen/X86/avx512fp16-mov.ll | 37 +-
llvm/test/CodeGen/X86/build-vector-128.ll | 62 +-
llvm/test/CodeGen/X86/build-vector-256.ll | 25 +-
llvm/test/CodeGen/X86/build-vector-512.ll | 243 +---
.../CodeGen/X86/buildvec-strided-loads.ll | 1128 +++++++++++++++++
.../X86/masked_gather_scatter_widen.ll | 252 ++--
llvm/test/CodeGen/X86/vector-compress.ll | 262 ++--
14 files changed, 2022 insertions(+), 2012 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/buildvec-strided-loads.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 46edb6c001955..64d298d26b84f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8065,6 +8065,102 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
}
}
+ // STRIDED - element loads at a uniform byte stride larger than the element
+ // size are folded into wide load(s) + vector truncation.
+ // Depth 1 lets the REVERSE block below recurse into us to catch reverse
+ // strides.
+ if (Depth <= 1 && Subtarget.hasAVX2() && !IsConsecutiveLoad &&
+ LoadMask.isAllOnes() && isPowerOf2_32(NumElems) && BaseSizeInBits <= 32 &&
+ VT.getSizeInBits() >= 128) {
+ unsigned WideEltBits = 0;
+ for (unsigned Trial : {16u, 32u, 64u}) {
+ if (Trial <= BaseSizeInBits || Trial % BaseSizeInBits != 0)
+ continue;
+ unsigned LaneStride = Trial / BaseSizeInBits;
+ bool AllMatch = true;
+ for (unsigned K = 1; K < NumElems && AllMatch; ++K) {
+ AllMatch = AllMatch && ByteOffsets[K] == 0 &&
+ DAG.areNonVolatileConsecutiveLoads(
+ Loads[K], LDBase, BaseSizeInBytes, K * LaneStride);
+ }
+ if (AllMatch) {
+ WideEltBits = Trial;
+ break;
+ }
+ }
+ if (WideEltBits != 0) {
+ MVT WideEltVT = MVT::getIntegerVT(WideEltBits);
+ MVT SrcEltVT = MVT::getIntegerVT(BaseSizeInBits);
+ // VTRUNC writes the truncated values to the low lanes of an xmm with
+ // zero padding above.
+ MVT TruncDstVT = MVT::getVectorVT(SrcEltVT, 128 / BaseSizeInBits);
+ unsigned TruncDstLanes = TruncDstVT.getVectorNumElements();
+ MVT ConcatVT = MVT::getVectorVT(SrcEltVT, NumElems);
+ if (NumElems > TruncDstLanes && !TLI.isTypeLegal(ConcatVT))
+ return SDValue();
+ // Try wider register sizes first.
+ for (unsigned WideRegBits : {512u, 256u, 128u}) {
+ unsigned LanesPerWideLoad = WideRegBits / WideEltBits;
+ if (LanesPerWideLoad < 2 || NumElems % LanesPerWideLoad != 0)
+ continue;
+ if (LanesPerWideLoad > TruncDstLanes)
+ continue; // VTRUNC dest must hold all good lanes of one piece.
+ MVT WideVT = MVT::getVectorVT(WideEltVT, LanesPerWideLoad);
+ if (!TLI.isTypeLegal(WideVT) || !TLI.isTypeLegal(TruncDstVT))
+ continue;
+ unsigned NumWideLoads = NumElems / LanesPerWideLoad;
+ // VTRUNC has no non-AVX-512 lowering so the i16 to i8 form needs BWI.
+ bool Partial = LanesPerWideLoad != TruncDstLanes;
+ if (Partial && (!Subtarget.hasAVX512() ||
+ (WideEltBits == 16 && !Subtarget.hasBWI())))
+ continue;
+ unsigned BytesPerWideLoad = WideRegBits / 8;
+ auto MMOFlags = LDBase->getMemOperand()->getFlags();
+ SDValue BasePtr = LDBase->getBasePtr();
+ SmallVector<SDValue, 8> Pieces;
+ for (unsigned K = 0; K != NumWideLoads; ++K) {
+ unsigned Offset = K * BytesPerWideLoad;
+ SDValue Ptr = K == 0 ? BasePtr
+ : DAG.getMemBasePlusOffset(
+ BasePtr, TypeSize::getFixed(Offset), DL);
+ SDValue Ld =
+ DAG.getLoad(WideVT, DL, LDBase->getChain(), Ptr,
+ LDBase->getPointerInfo().getWithOffset(Offset),
+ K == 0 ? LDBase->getBaseAlign() : Align(1), MMOFlags);
+ for (auto *LD : Loads)
+ if (LD)
+ DAG.makeEquivalentMemoryOrdering(LD, Ld);
+ unsigned TruncOp = Partial ? X86ISD::VTRUNC : ISD::TRUNCATE;
+ Pieces.push_back(DAG.getNode(TruncOp, DL, TruncDstVT, Ld));
+ }
+ // Pairwise shuffle the low halves until each piece is full.
+ if (Partial) {
+ unsigned GoodLanes = LanesPerWideLoad;
+ while (Pieces.size() > 1 && GoodLanes < TruncDstLanes) {
+ assert((TruncDstLanes % GoodLanes) == 0 &&
+ "Illegal VTRUNC packing");
+ SmallVector<SDValue, 8> Next;
+ SmallVector<int, 16> Mask(TruncDstLanes, -1);
+ for (unsigned I = 0; I != GoodLanes; ++I) {
+ Mask[I] = I;
+ Mask[GoodLanes + I] = TruncDstLanes + I;
+ }
+ for (unsigned J = 0, E = Pieces.size() - 1; J < E; J += 2)
+ Next.push_back(DAG.getVectorShuffle(TruncDstVT, DL, Pieces[J],
+ Pieces[J + 1], Mask));
+ Pieces = std::move(Next);
+ GoodLanes *= 2;
+ }
+ }
+ if (Pieces.size() == 1)
+ return DAG.getBitcast(VT, Pieces[0]);
+
+ SDValue Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Pieces);
+ return DAG.getBitcast(VT, Result);
+ }
+ }
+ }
+
// REVERSE - attempt to match the loads in reverse and then shuffle back.
// TODO: Do this for any permute or mismatching element counts.
if (Depth == 0 && ZeroMask.isZero() && UndefMask.isZero() &&
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index 0952647919b5e..2e7425c13009f 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -679,47 +679,36 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; KNL-NEXT: pushq %r13
; KNL-NEXT: pushq %r12
; KNL-NEXT: pushq %rbx
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 160(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, 168(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, 176(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, 184(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, 192(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, 200(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, 208(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, 216(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, 224(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, 232(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, 240(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, 248(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, 256(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, 264(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, 272(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: movq %rdi, %rax
-; KNL-NEXT: vmovd %esi, %xmm2
-; KNL-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, 56(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, 64(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, 72(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, 80(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, 88(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, 96(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, 104(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, 112(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, 120(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, 128(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, 136(%rsp), %xmm2, %xmm2
+; KNL-NEXT: vmovdqu64 152(%rsp), %zmm0
+; KNL-NEXT: vmovdqu64 216(%rsp), %zmm1
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; KNL-NEXT: vmovd %esi, %xmm1
+; KNL-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $5, 56(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $6, 64(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $7, 72(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $8, 80(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $9, 88(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $10, 96(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $11, 104(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $12, 112(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $13, 120(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $14, 128(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $15, 136(%rsp), %xmm1, %xmm1
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: kmovw 280(%rsp), %k0
; KNL-NEXT: kmovw 144(%rsp), %k1
; KNL-NEXT: kandw %k0, %k1, %k1
-; KNL-NEXT: vptestmd %zmm1, %zmm0, %k2
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; KNL-NEXT: vptestmd %zmm1, %zmm0, %k0 {%k2}
+; KNL-NEXT: vptestmd %zmm2, %zmm0, %k2
+; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vptestmd %zmm2, %zmm0, %k0 {%k2}
; KNL-NEXT: kmovw %k1, %edx
; KNL-NEXT: kshiftrw $1, %k0, %k1
; KNL-NEXT: kmovw %k1, %r9d
@@ -815,6 +804,15 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; SKX-NEXT: pushq %r12
; SKX-NEXT: pushq %rbx
; SKX-NEXT: movq %rdi, %rax
+; SKX-NEXT: vmovdqu64 152(%rsp), %zmm0
+; SKX-NEXT: vmovdqu64 216(%rsp), %zmm1
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vpmovqb %zmm0, %xmm0
+; SKX-NEXT: vpbroadcastb 280(%rsp), %xmm2
+; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SKX-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SKX-NEXT: vptestmb %ymm1, %ymm0, %k1
; SKX-NEXT: vmovd %esi, %xmm0
; SKX-NEXT: vpinsrb $1, %edx, %xmm0, %xmm0
; SKX-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0
@@ -831,29 +829,9 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; SKX-NEXT: vpinsrb $13, 120(%rsp), %xmm0, %xmm0
; SKX-NEXT: vpinsrb $14, 128(%rsp), %xmm0, %xmm0
; SKX-NEXT: vpinsrb $15, 136(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpbroadcastb 144(%rsp), %xmm1
-; SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 160(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, 168(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, 176(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, 184(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, 192(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 200(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 208(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 216(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 224(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 232(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 240(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 248(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 256(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 264(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; SKX-NEXT: vpinsrb $15, 272(%rsp), %xmm2, %xmm1
-; SKX-NEXT: vpbroadcastb 280(%rsp), %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; SKX-NEXT: vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SKX-NEXT: vptestmb %ymm2, %ymm1, %k1
-; SKX-NEXT: vptestmb %ymm2, %ymm0, %k0 {%k1}
+; SKX-NEXT: vpbroadcastb 144(%rsp), %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SKX-NEXT: vptestmb %ymm1, %ymm0, %k0 {%k1}
; SKX-NEXT: kshiftrd $16, %k0, %k1
; SKX-NEXT: kmovd %k1, %edx
; SKX-NEXT: kshiftrd $1, %k0, %k1
@@ -948,47 +926,13 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; KNL_X32-NEXT: pushl %ebx
; KNL_X32-NEXT: pushl %edi
; KNL_X32-NEXT: pushl %esi
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 96(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $2, 100(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $3, 104(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $4, 108(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $5, 112(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $6, 116(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $7, 120(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $8, 124(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $9, 128(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $10, 132(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $11, 136(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $12, 140(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $13, 144(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $14, 148(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $15, 152(%esp), %xmm0, %xmm1
; KNL_X32-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL_X32-NEXT: kmovw 156(%esp), %k0
; KNL_X32-NEXT: kmovw 88(%esp), %k1
; KNL_X32-NEXT: kandw %k0, %k1, %k1
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k2
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 28(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $2, 32(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $3, 36(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $4, 40(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $5, 44(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $6, 48(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $7, 52(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $8, 56(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $9, 60(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $10, 64(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $11, 68(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $12, 72(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $13, 76(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $14, 80(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $15, 84(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL_X32-NEXT: vptestmd 92(%esp), %zmm0, %k2
; KNL_X32-NEXT: movl 20(%esp), %eax
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k0 {%k2}
+; KNL_X32-NEXT: vptestmd 24(%esp), %zmm0, %k0 {%k2}
; KNL_X32-NEXT: kmovw %k1, %ebx
; KNL_X32-NEXT: kshiftrw $1, %k0, %k1
; KNL_X32-NEXT: kmovw %k1, %ebp
@@ -1081,23 +1025,12 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; FASTISEL-NEXT: pushq %r13
; FASTISEL-NEXT: pushq %r12
; FASTISEL-NEXT: pushq %rbx
+; FASTISEL-NEXT: vmovdqu64 152(%rsp), %zmm0
+; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm1
+; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
; FASTISEL-NEXT: movq %rdi, %rax
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 160(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $2, 168(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $3, 176(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $4, 184(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $5, 192(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $6, 200(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $7, 208(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $8, 216(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $9, 224(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $10, 232(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $11, 240(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $12, 248(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $13, 256(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $14, 264(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $15, 272(%rsp), %xmm0, %xmm0
+; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
+; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; FASTISEL-NEXT: vpbroadcastb 280(%rsp), %xmm1
; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; FASTISEL-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
@@ -2365,312 +2298,161 @@ define void @v64i1_mem(<128 x i32> %x, <64 x i1> %y) {
define i128 @PR179334(<128 x i1> %m) nounwind {
; KNL-LABEL: PR179334:
; KNL: ## %bb.0:
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 96(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, 104(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, 112(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, 120(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, 128(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, 136(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, 144(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, 152(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, 160(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, 168(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, 176(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, 184(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, 192(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, 200(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, 208(%rsp), %xmm0, %xmm0
+; KNL-NEXT: pushq %rbx
+; KNL-NEXT: vmovdqu64 480(%rsp), %zmm0
+; KNL-NEXT: vmovdqu64 544(%rsp), %zmm1
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
; KNL-NEXT: kmovw %k0, %eax
-; KNL-NEXT: vmovd %edi, %xmm1
-; KNL-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 8(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 16(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 24(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 32(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 40(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 48(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 56(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 64(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 72(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 80(%rsp), %xmm1, %xmm1
-; KNL-NEXT: shll $16, %eax
+; KNL-NEXT: vmovdqu64 608(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 672(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %ecx
-; KNL-NEXT: orl %eax, %ecx
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 224(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 232(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 240(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 248(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 256(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 264(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 272(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 280(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 288(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 296(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 304(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 312(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 320(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 328(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 336(%rsp), %xmm1, %xmm1
+; KNL-NEXT: kmovw %k0, %r11d
+; KNL-NEXT: shll $16, %r11d
+; KNL-NEXT: orl %eax, %r11d
+; KNL-NEXT: vmovdqu64 736(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 800(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %edx
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 352(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 360(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 368(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 376(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 384(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 392(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 400(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 408(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 416(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 424(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 432(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 440(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 448(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 456(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 464(%rsp), %xmm1, %xmm1
+; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: vmovdqu64 864(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 928(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %eax
-; KNL-NEXT: shll $16, %eax
-; KNL-NEXT: orl %edx, %eax
-; KNL-NEXT: shlq $32, %rax
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 480(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 488(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 496(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 504(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 512(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 520(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 528(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 536(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 544(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 552(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 560(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 568(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 576(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 584(%rsp), %xmm1, %xmm1
-; KNL-NEXT: orq %rcx, %rax
-; KNL-NEXT: vpinsrb $15, 592(%rsp), %xmm1, %xmm1
+; KNL-NEXT: kmovw %k0, %r10d
+; KNL-NEXT: shll $16, %r10d
+; KNL-NEXT: orl %eax, %r10d
+; KNL-NEXT: shlq $32, %r10
+; KNL-NEXT: orq %r11, %r10
+; KNL-NEXT: vmovdqu64 224(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 288(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 608(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 616(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 624(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 632(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 640(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 648(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 656(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 664(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 672(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 680(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 688(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 696(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 704(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 712(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 720(%rsp), %xmm1, %xmm1
-; KNL-NEXT: kmovw %k0, %edx
+; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: vmovdqu64 352(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 416(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %ecx
-; KNL-NEXT: shll $16, %ecx
-; KNL-NEXT: orl %edx, %ecx
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 736(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 744(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 752(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 760(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 768(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 776(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 784(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 792(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 800(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 808(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 816(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 824(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 832(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 840(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 848(%rsp), %xmm1, %xmm1
+; KNL-NEXT: kmovw %k0, %r11d
+; KNL-NEXT: shll $16, %r11d
+; KNL-NEXT: orl %eax, %r11d
+; KNL-NEXT: shlq $32, %r11
+; KNL-NEXT: vmovdqu64 96(%rsp), %zmm1
+; KNL-NEXT: vmovdqu64 160(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %esi
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 864(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 872(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 880(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 888(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 896(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 904(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 912(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 920(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 928(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 936(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 944(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 952(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 960(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 968(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 976(%rsp), %xmm1, %xmm1
+; KNL-NEXT: kmovw %k0, %ebx
+; KNL-NEXT: vmovd %edi, %xmm1
+; KNL-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $6, 16(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $7, 24(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $8, 32(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $9, 40(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $10, 48(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $11, 56(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $12, 64(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $13, 72(%rsp), %xmm1, %xmm1
+; KNL-NEXT: vpinsrb $14, 80(%rsp), %xmm1, %xmm1
+; KNL-NEXT: shll $16, %ebx
+; KNL-NEXT: vpinsrb $15, 88(%rsp), %xmm1, %xmm1
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %edx
-; KNL-NEXT: shll $16, %edx
-; KNL-NEXT: orl %esi, %edx
-; KNL-NEXT: shlq $32, %rdx
-; KNL-NEXT: orq %rcx, %rdx
+; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: orl %ebx, %eax
+; KNL-NEXT: orq %r11, %rax
+; KNL-NEXT: movq %r10, %rdx
+; KNL-NEXT: popq %rbx
; KNL-NEXT: retq
;
; SKX-LABEL: PR179334:
; SKX: ## %bb.0:
-; SKX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 224(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $2, 232(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $3, 240(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $4, 248(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $5, 256(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $6, 264(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $7, 272(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $8, 280(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $9, 288(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $10, 296(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $11, 304(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $12, 312(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $13, 320(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $14, 328(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $15, 336(%rsp), %xmm0, %xmm0
-; SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 352(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, 360(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, 368(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, 376(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, 384(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 392(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 400(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 408(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 416(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 424(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 432(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 440(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 448(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 456(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $15, 464(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vmovd %edi, %xmm2
-; SKX-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 8(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 16(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 24(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 32(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 40(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 48(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 56(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 64(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 72(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $15, 80(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 96(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $2, 104(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $3, 112(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $4, 120(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $5, 128(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $6, 136(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $7, 144(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $8, 152(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $9, 160(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $10, 168(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $11, 176(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $12, 184(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $13, 192(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $14, 200(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $15, 208(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; SKX-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm1
-; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm1
-; SKX-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SKX-NEXT: vptestmb %zmm0, %zmm1, %k0
-; SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 736(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, 744(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, 752(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, 760(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, 768(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 776(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 784(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 792(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 800(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 808(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 816(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 824(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 832(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 840(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $15, 848(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 864(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, 872(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, 880(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, 888(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, 896(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 904(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 912(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 920(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 928(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 936(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 944(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 952(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 960(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 968(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $15, 976(%rsp), %xmm2, %xmm2
-; SKX-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 480(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $2, 488(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $3, 496(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $4, 504(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $5, 512(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $6, 520(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $7, 528(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $8, 536(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $9, 544(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $10, 552(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $11, 560(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $12, 568(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $13, 576(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $14, 584(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vpinsrb $15, 592(%rsp), %xmm3, %xmm3
-; SKX-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 608(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $2, 616(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $3, 624(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $4, 632(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $5, 640(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $6, 648(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $7, 656(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $8, 664(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $9, 672(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $10, 680(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $11, 688(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $12, 696(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $13, 704(%rsp), %xmm4, %xmm4
-; SKX-NEXT: vpinsrb $14, 712(%rsp), %xmm4, %xmm4
+; SKX-NEXT: vmovdqu64 216(%rsp), %zmm0
+; SKX-NEXT: vmovdqu64 280(%rsp), %zmm1
+; SKX-NEXT: vmovdqu64 344(%rsp), %zmm2
+; SKX-NEXT: vmovdqu64 408(%rsp), %zmm3
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vpmovqb %zmm3, %xmm3
+; SKX-NEXT: vpmovqb %zmm0, %xmm0
+; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; SKX-NEXT: vpmovqb %zmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SKX-NEXT: vmovdqu64 88(%rsp), %zmm2
+; SKX-NEXT: vmovdqu64 152(%rsp), %zmm3
+; SKX-NEXT: vpmovqb %zmm3, %xmm3
+; SKX-NEXT: vpmovqb %zmm2, %xmm2
+; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; SKX-NEXT: vmovd %edi, %xmm1
+; SKX-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $6, 8(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $7, 16(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $8, 24(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $9, 32(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $10, 40(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $11, 48(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $12, 56(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $13, 64(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpinsrb $14, 72(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SKX-NEXT: vpinsrb $15, 80(%rsp), %xmm1, %xmm1
; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; SKX-NEXT: vpinsrb $15, 720(%rsp), %xmm4, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; SKX-NEXT: vptestmb %zmm0, %zmm1, %k1
+; SKX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; SKX-NEXT: vptestmb %zmm2, %zmm0, %k0
+; SKX-NEXT: vmovdqu64 792(%rsp), %zmm0
+; SKX-NEXT: vpmovqb %zmm0, %xmm0
+; SKX-NEXT: vmovdqu64 920(%rsp), %zmm1
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vmovdqu64 472(%rsp), %zmm3
+; SKX-NEXT: vmovdqu64 536(%rsp), %zmm4
+; SKX-NEXT: vmovdqu64 600(%rsp), %zmm5
+; SKX-NEXT: vmovdqu64 664(%rsp), %zmm6
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT: vpmovqb %zmm4, %xmm1
+; SKX-NEXT: vpmovqb %zmm6, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
+; SKX-NEXT: vmovdqu64 728(%rsp), %zmm4
+; SKX-NEXT: vpmovqb %zmm4, %xmm4
+; SKX-NEXT: vmovdqu64 856(%rsp), %zmm6
+; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; SKX-NEXT: vpmovqb %zmm6, %xmm1
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm4, %ymm1
+; SKX-NEXT: vpmovqb %zmm3, %xmm3
+; SKX-NEXT: vpmovqb %zmm5, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; SKX-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; SKX-NEXT: vptestmb %zmm2, %zmm0, %k1
; SKX-NEXT: kmovq %k1, %rdx
; SKX-NEXT: kmovq %k0, %rax
; SKX-NEXT: vzeroupper
@@ -2678,151 +2460,15 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
;
; KNL_X32-LABEL: PR179334:
; KNL_X32: ## %bb.0:
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 12(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $2, 16(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $3, 20(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $4, 24(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $5, 28(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $6, 32(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $7, 36(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $8, 40(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $9, 44(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $10, 48(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $11, 52(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $12, 56(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $13, 60(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $14, 64(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpinsrb $15, 68(%esp), %xmm0, %xmm0
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL_X32-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 76(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $2, 80(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $3, 84(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $4, 88(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $5, 92(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $6, 96(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $7, 100(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $8, 104(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $9, 108(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $10, 112(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $11, 116(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $12, 120(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $13, 124(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $14, 128(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL_X32-NEXT: vpinsrb $15, 132(%esp), %xmm2, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k1
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 140(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $2, 144(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $3, 148(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $4, 152(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $5, 156(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $6, 160(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $7, 164(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $8, 168(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $9, 172(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $10, 176(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $11, 180(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $12, 184(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $13, 188(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $14, 192(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $15, 196(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 204(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $2, 208(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $3, 212(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $4, 216(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $5, 220(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $6, 224(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $7, 228(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $8, 232(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $9, 236(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $10, 240(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $11, 244(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $12, 248(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $13, 252(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $14, 256(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $15, 260(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k2
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k3
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 268(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $2, 272(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $3, 276(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $4, 280(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $5, 284(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $6, 288(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $7, 292(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $8, 296(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $9, 300(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $10, 304(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $11, 308(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $12, 312(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $13, 316(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $14, 320(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $15, 324(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 332(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $2, 336(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $3, 340(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $4, 344(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $5, 348(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $6, 352(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $7, 356(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $8, 360(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $9, 364(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $10, 368(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $11, 372(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $12, 376(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $13, 380(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $14, 384(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k4
-; KNL_X32-NEXT: vpinsrb $15, 388(%esp), %xmm2, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k5
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 396(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $2, 400(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $3, 404(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $4, 408(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $5, 412(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $6, 416(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $7, 420(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $8, 424(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $9, 428(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $10, 432(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $11, 436(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $12, 440(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $13, 444(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $14, 448(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpinsrb $15, 452(%esp), %xmm1, %xmm1
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; KNL_X32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL_X32-NEXT: vpinsrb $1, 460(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $2, 464(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $3, 468(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $4, 472(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $5, 476(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $6, 480(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $7, 484(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $8, 488(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $9, 492(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $10, 496(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $11, 500(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $12, 504(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $13, 508(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $14, 512(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vpinsrb $15, 516(%esp), %xmm2, %xmm2
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k6
-; KNL_X32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; KNL_X32-NEXT: vptestmd %zmm0, %zmm1, %k7
+; KNL_X32-NEXT: vptestmd 8(%esp), %zmm0, %k0
+; KNL_X32-NEXT: vptestmd 72(%esp), %zmm0, %k1
+; KNL_X32-NEXT: vptestmd 136(%esp), %zmm0, %k2
+; KNL_X32-NEXT: vptestmd 200(%esp), %zmm0, %k3
+; KNL_X32-NEXT: vptestmd 264(%esp), %zmm0, %k4
+; KNL_X32-NEXT: vptestmd 328(%esp), %zmm0, %k5
+; KNL_X32-NEXT: vptestmd 392(%esp), %zmm0, %k6
+; KNL_X32-NEXT: vptestmd 456(%esp), %zmm0, %k7
; KNL_X32-NEXT: movl 4(%esp), %eax
; KNL_X32-NEXT: kmovw %k7, 14(%eax)
; KNL_X32-NEXT: kmovw %k6, 12(%eax)
@@ -2836,143 +2482,66 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
;
; FASTISEL-LABEL: PR179334:
; FASTISEL: ## %bb.0:
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 736(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $2, 744(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $3, 752(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $4, 760(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $5, 768(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $6, 776(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $7, 784(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $8, 792(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $9, 800(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $10, 808(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $11, 816(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $12, 824(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $13, 832(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $14, 840(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vpinsrb $15, 848(%rsp), %xmm0, %xmm0
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 864(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $2, 872(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $3, 880(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $4, 888(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $5, 896(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $6, 904(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $7, 912(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $8, 920(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $9, 928(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $10, 936(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $11, 944(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $12, 952(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $13, 960(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $14, 968(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $15, 976(%rsp), %xmm1, %xmm1
+; FASTISEL-NEXT: vmovdqu64 792(%rsp), %zmm0
+; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
+; FASTISEL-NEXT: vmovdqu64 920(%rsp), %zmm1
+; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
+; FASTISEL-NEXT: vmovdqu64 472(%rsp), %zmm2
+; FASTISEL-NEXT: vmovdqu64 536(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 600(%rsp), %zmm4
+; FASTISEL-NEXT: vmovdqu64 664(%rsp), %zmm5
+; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
+; FASTISEL-NEXT: vpmovqb %zmm5, %xmm5
; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 480(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $2, 488(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $3, 496(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $4, 504(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $5, 512(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $6, 520(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $7, 528(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $8, 536(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $9, 544(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $10, 552(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $11, 560(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $12, 568(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $13, 576(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $14, 584(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vpinsrb $15, 592(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 608(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $2, 616(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $3, 624(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $4, 632(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $5, 640(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $6, 648(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $7, 656(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $8, 664(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $9, 672(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $10, 680(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $11, 688(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $12, 696(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $13, 704(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $14, 712(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $15, 720(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm1
-; FASTISEL-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 224(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $2, 232(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $3, 240(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $4, 248(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $5, 256(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $6, 264(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $7, 272(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $8, 280(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $9, 288(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $10, 296(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $11, 304(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $12, 312(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $13, 320(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $14, 328(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpinsrb $15, 336(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 352(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $2, 360(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $3, 368(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $4, 376(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $5, 384(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $6, 392(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $7, 400(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $8, 408(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $9, 416(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $10, 424(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $11, 432(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $12, 440(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $13, 448(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vptestmb %zmm0, %zmm1, %k0
-; FASTISEL-NEXT: vpinsrb $14, 456(%rsp), %xmm3, %xmm1
-; FASTISEL-NEXT: vpinsrb $15, 464(%rsp), %xmm1, %xmm1
-; FASTISEL-NEXT: vmovd %edi, %xmm3
-; FASTISEL-NEXT: vpinsrb $1, %esi, %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $3, %ecx, %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $4, %r8d, %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $5, %r9d, %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $6, 8(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $7, 16(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $8, 24(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $9, 32(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $10, 40(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $11, 48(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $12, 56(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $13, 64(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $14, 72(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vpinsrb $15, 80(%rsp), %xmm3, %xmm3
-; FASTISEL-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; FASTISEL-NEXT: vpinsrb $1, 96(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $2, 104(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $3, 112(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $4, 120(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $5, 128(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $6, 136(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $7, 144(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $8, 152(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $9, 160(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $10, 168(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $11, 176(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $12, 184(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $13, 192(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vpinsrb $14, 200(%rsp), %xmm4, %xmm4
-; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; FASTISEL-NEXT: vpinsrb $15, 208(%rsp), %xmm4, %xmm2
-; FASTISEL-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; FASTISEL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm1
+; FASTISEL-NEXT: vmovdqu64 728(%rsp), %zmm3
+; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; FASTISEL-NEXT: vmovdqu64 856(%rsp), %zmm1
+; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
+; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; FASTISEL-NEXT: vpmovqb %zmm2, %xmm2
+; FASTISEL-NEXT: vpmovqb %zmm4, %xmm3
+; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; FASTISEL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; FASTISEL-NEXT: vptestmb %zmm0, %zmm1, %k1
+; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; FASTISEL-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k0
+; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm0
+; FASTISEL-NEXT: vmovdqu64 280(%rsp), %zmm2
+; FASTISEL-NEXT: vmovdqu64 344(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 408(%rsp), %zmm4
+; FASTISEL-NEXT: vpmovqb %zmm2, %xmm2
+; FASTISEL-NEXT: vpmovqb %zmm4, %xmm4
+; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
+; FASTISEL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
+; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; FASTISEL-NEXT: vmovdqu64 88(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 152(%rsp), %zmm4
+; FASTISEL-NEXT: vpmovqb %zmm4, %xmm4
+; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
+; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; FASTISEL-NEXT: vmovd %edi, %xmm2
+; FASTISEL-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $6, 8(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $7, 16(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $8, 24(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $9, 32(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $10, 40(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $11, 48(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $12, 56(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $13, 64(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpinsrb $14, 72(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; FASTISEL-NEXT: vpinsrb $15, 80(%rsp), %xmm2, %xmm2
+; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k1
; FASTISEL-NEXT: kmovq %k1, %rax
; FASTISEL-NEXT: kmovq %k0, %rdx
; FASTISEL-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index 5aad782c7134e..b4ccc1b0f610d 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -1863,22 +1863,11 @@ define void @extload_v8i64(ptr %a, ptr %res) {
define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; KNL-LABEL: test21:
; KNL: # %bb.0:
-; KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; KNL-NEXT: vpmovqb %zmm3, %xmm3
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k1
@@ -1900,40 +1889,18 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k2
-; KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; KNL-NEXT: vpmovqb %zmm4, %xmm4
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k3
-; KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; KNL-NEXT: vpmovqb %zmm4, %xmm4
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k4
; KNL-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
@@ -1961,22 +1928,11 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
;
; AVX512DQNOBW-LABEL: test21:
; AVX512DQNOBW: # %bb.0:
-; AVX512DQNOBW-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512DQNOBW-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; AVX512DQNOBW-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; AVX512DQNOBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k0
@@ -1998,40 +1954,18 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; AVX512DQNOBW-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k1
-; AVX512DQNOBW-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512DQNOBW-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; AVX512DQNOBW-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k2
-; AVX512DQNOBW-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512DQNOBW-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; AVX512DQNOBW-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k3
; AVX512DQNOBW-NEXT: vpmovm2d %k3, %zmm2
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index 4b44afb33d103..c905082b42233 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -1732,110 +1732,55 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
; KNL-NEXT: subq $192, %rsp
; KNL-NEXT: movl 744(%rbp), %eax
; KNL-NEXT: andl $127, %eax
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 232(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, 240(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, 248(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, 256(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, 264(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, 272(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, 280(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, 288(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, 296(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, 304(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, 312(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, 320(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, 328(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, 336(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, 344(%rbp), %xmm0, %xmm0
-; KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 360(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $2, 368(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $3, 376(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $4, 384(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $5, 392(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $6, 400(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $7, 408(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $8, 416(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $9, 424(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $10, 432(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $11, 440(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $12, 448(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $13, 456(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $14, 464(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vpinsrb $15, 472(%rbp), %xmm1, %xmm1
-; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; KNL-NEXT: vmovdqu64 224(%rbp), %zmm0
+; KNL-NEXT: vmovdqu64 288(%rbp), %zmm1
+; KNL-NEXT: vmovdqu64 352(%rbp), %zmm2
+; KNL-NEXT: vmovdqu64 416(%rbp), %zmm3
+; KNL-NEXT: vpmovqb %zmm1, %xmm1
+; KNL-NEXT: vpmovqb %zmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
; KNL-NEXT: vpcmpeqb %ymm0, %ymm1, %ymm1
-; KNL-NEXT: vmovd %edi, %xmm2
-; KNL-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, 16(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, 24(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, 32(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, 40(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, 48(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, 56(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, 64(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, 72(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, 80(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, 88(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 104(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $2, 112(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $3, 120(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $4, 128(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $5, 136(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $6, 144(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $7, 152(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $8, 160(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $9, 168(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $10, 176(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $11, 184(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $12, 192(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $13, 200(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $14, 208(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $15, 216(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; KNL-NEXT: vmovdqu64 96(%rbp), %zmm2
+; KNL-NEXT: vmovdqu64 160(%rbp), %zmm3
+; KNL-NEXT: vpmovqb %zmm3, %xmm3
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; KNL-NEXT: vmovd %edi, %xmm3
+; KNL-NEXT: vpinsrb $1, %esi, %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $3, %ecx, %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $4, %r8d, %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $5, %r9d, %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $6, 16(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $7, 24(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $8, 32(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $9, 40(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $10, 48(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $11, 56(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $12, 64(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $13, 72(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $14, 80(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vpinsrb $15, 88(%rbp), %xmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2
; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; KNL-NEXT: vpternlogq {{.*#+}} zmm1 = ~zmm1
-; KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 488(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $2, 496(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $3, 504(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $4, 512(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $5, 520(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $6, 528(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $7, 536(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $8, 544(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $9, 552(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $10, 560(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $11, 568(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $12, 576(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $13, 584(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $14, 592(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vpinsrb $15, 600(%rbp), %xmm2, %xmm2
-; KNL-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, 616(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $2, 624(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $3, 632(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $4, 640(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $5, 648(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $6, 656(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $7, 664(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $8, 672(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $9, 680(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $10, 688(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $11, 696(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $12, 704(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $13, 712(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $14, 720(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $15, 728(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; KNL-NEXT: vmovdqu64 480(%rbp), %zmm2
+; KNL-NEXT: vmovdqu64 544(%rbp), %zmm3
+; KNL-NEXT: vmovdqu64 608(%rbp), %zmm4
+; KNL-NEXT: vmovdqu64 672(%rbp), %zmm5
+; KNL-NEXT: vpmovqb %zmm3, %xmm3
+; KNL-NEXT: vpmovqb %zmm5, %xmm5
+; KNL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm4, %xmm4
+; KNL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; KNL-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2]
; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2
; KNL-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2
; KNL-NEXT: cmpb $0, 736(%rbp)
@@ -1898,106 +1843,51 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
; SKX-NEXT: movq %rsp, %rbp
; SKX-NEXT: andq $-64, %rsp
; SKX-NEXT: subq $192, %rsp
-; SKX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 232(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $2, 240(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $3, 248(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $4, 256(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $5, 264(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $6, 272(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $7, 280(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $8, 288(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $9, 296(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $10, 304(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $11, 312(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $12, 320(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $13, 328(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $14, 336(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vpinsrb $15, 344(%rbp), %xmm0, %xmm0
-; SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 360(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, 368(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, 376(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, 384(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, 392(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 400(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 408(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 416(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 424(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 432(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 440(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 448(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 456(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 464(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $15, 472(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; SKX-NEXT: vmovd %edi, %xmm1
-; SKX-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 16(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 24(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 32(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 40(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 48(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 56(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 64(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 72(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 80(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $15, 88(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 104(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, 112(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, 120(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, 128(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, 136(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 144(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 152(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 160(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 168(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 176(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 184(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 192(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 200(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 208(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $15, 216(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vmovdqu64 224(%rbp), %zmm0
+; SKX-NEXT: vmovdqu64 288(%rbp), %zmm1
+; SKX-NEXT: vmovdqu64 352(%rbp), %zmm2
+; SKX-NEXT: vmovdqu64 416(%rbp), %zmm3
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vpmovqb %zmm3, %xmm3
+; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; SKX-NEXT: vpmovqb %zmm0, %xmm0
+; SKX-NEXT: vpmovqb %zmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; SKX-NEXT: vmovdqu64 96(%rbp), %zmm1
+; SKX-NEXT: vmovdqu64 160(%rbp), %zmm2
+; SKX-NEXT: vpmovqb %zmm2, %xmm2
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SKX-NEXT: vmovd %edi, %xmm2
+; SKX-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $6, 16(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $7, 24(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $8, 32(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $9, 40(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $10, 48(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $11, 56(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $12, 64(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $13, 72(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $14, 80(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vpinsrb $15, 88(%rbp), %xmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 488(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, 496(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, 504(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, 512(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, 520(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 528(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 536(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 544(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 552(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 560(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 568(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 576(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 584(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 592(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $15, 600(%rbp), %xmm1, %xmm1
-; SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SKX-NEXT: vpinsrb $1, 616(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, 624(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, 632(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, 640(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, 648(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 656(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 664(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 672(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 680(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 688(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 696(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 704(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 712(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 720(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $15, 728(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vmovdqu64 480(%rbp), %zmm1
+; SKX-NEXT: vmovdqu64 544(%rbp), %zmm2
+; SKX-NEXT: vmovdqu64 608(%rbp), %zmm3
+; SKX-NEXT: vmovdqu64 672(%rbp), %zmm4
+; SKX-NEXT: vpmovqb %zmm2, %xmm2
+; SKX-NEXT: vpmovqb %zmm4, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; SKX-NEXT: vpmovqb %zmm1, %xmm1
+; SKX-NEXT: vpmovqb %zmm3, %xmm3
+; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
; SKX-NEXT: movl 744(%rbp), %eax
; SKX-NEXT: andl $127, %eax
; SKX-NEXT: vptestmb %zmm0, %zmm0, %k0
diff --git a/llvm/test/CodeGen/X86/avx512-load-store.ll b/llvm/test/CodeGen/X86/avx512-load-store.ll
index f0677985621e1..fef164c87b44b 100644
--- a/llvm/test/CodeGen/X86/avx512-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx512-load-store.ll
@@ -374,10 +374,39 @@ define <80 x i32> @test_maskz_load_v80i32(ptr %p, <80 x i1> %mask) nounwind {
; CHECK64-LABEL: test_maskz_load_v80i32:
; CHECK64: # %bb.0:
; CHECK64-NEXT: movq %rdi, %rax
-; CHECK64-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; CHECK64-NEXT: vpmovqb %zmm1, %xmm1
+; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k1
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k2
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k3
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k4
+; CHECK64-NEXT: vmovd %edx, %xmm0
+; CHECK64-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0
+; CHECK64-NEXT: vpinsrb $2, %r8d, %xmm0, %xmm0
+; CHECK64-NEXT: vpinsrb $3, %r9d, %xmm0, %xmm0
; CHECK64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; CHECK64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; CHECK64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
@@ -390,81 +419,8 @@ define <80 x i32> @test_maskz_load_v80i32(ptr %p, <80 x i1> %mask) nounwind {
; CHECK64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; CHECK64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; CHECK64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; CHECK64-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK64-NEXT: vptestmd %zmm0, %zmm1, %k1
-; CHECK64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK64-NEXT: vptestmd %zmm0, %zmm1, %k2
-; CHECK64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK64-NEXT: vptestmd %zmm0, %zmm1, %k3
-; CHECK64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK64-NEXT: vptestmd %zmm0, %zmm1, %k4
-; CHECK64-NEXT: vmovd %edx, %xmm1
-; CHECK64-NEXT: vpinsrb $1, %ecx, %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $2, %r8d, %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $3, %r9d, %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK64-NEXT: vptestmd %zmm0, %zmm1, %k5
+; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k5
; CHECK64-NEXT: vmovdqu32 (%rsi), %zmm0 {%k5} {z}
; CHECK64-NEXT: vmovdqu32 64(%rsi), %zmm1 {%k4} {z}
; CHECK64-NEXT: vmovdqu32 128(%rsi), %zmm2 {%k3} {z}
@@ -480,97 +436,12 @@ define <80 x i32> @test_maskz_load_v80i32(ptr %p, <80 x i1> %mask) nounwind {
;
; CHECK32-LABEL: test_maskz_load_v80i32:
; CHECK32: # %bb.0:
-; CHECK32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; CHECK32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; CHECK32-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK32-NEXT: vptestmd %zmm0, %zmm1, %k1
-; CHECK32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK32-NEXT: vptestmd %zmm0, %zmm1, %k2
-; CHECK32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK32-NEXT: vptestmd %zmm0, %zmm1, %k3
-; CHECK32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK32-NEXT: vptestmd %zmm0, %zmm1, %k4
-; CHECK32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; CHECK32-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; CHECK32-NEXT: vptestmd %zmm0, %zmm1, %k5
+; CHECK32-NEXT: vptestmd {{[0-9]+}}(%esp), %zmm0, %k1
+; CHECK32-NEXT: vptestmd {{[0-9]+}}(%esp), %zmm0, %k2
+; CHECK32-NEXT: vptestmd {{[0-9]+}}(%esp), %zmm0, %k3
+; CHECK32-NEXT: vptestmd {{[0-9]+}}(%esp), %zmm0, %k4
+; CHECK32-NEXT: vptestmd {{[0-9]+}}(%esp), %zmm0, %k5
; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK32-NEXT: vmovdqu32 (%ecx), %zmm0 {%k5} {z}
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index db53dda401bc2..57eaadf916960 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -2914,58 +2914,25 @@ define void @store_64i1(ptr %a, <64 x i1> %v) {
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k0
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k1
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k2
-; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; KNL-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; KNL-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; KNL-NEXT: vpmovqb %zmm2, %xmm2
+; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k3
; KNL-NEXT: kmovw %k3, 6(%rdi)
@@ -3012,58 +2979,25 @@ define void @store_64i1(ptr %a, <64 x i1> %v) {
; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k0
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQ-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k1
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQ-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k2
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX512DQ-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX512DQ-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k3
; AVX512DQ-NEXT: kmovw %k3, 6(%rdi)
diff --git a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
index e92c9e66d0fa5..b74771c825780 100644
--- a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
@@ -183,60 +183,27 @@ declare void @llvm.masked.store.v16f16.p0(<16 x half>, ptr, i32, <16 x i1>)
define void @test_maskz_store_v192i8(ptr %p0, <192 x i1> %mask) nounwind {
; CHECK-LABEL: test_maskz_store_v192i8:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; CHECK-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT: vmovd %esi, %xmm1
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vpmovqb %zmm1, %xmm1
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; CHECK-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vpmovqb %zmm1, %xmm1
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; CHECK-NEXT: vmovd %esi, %xmm2
+; CHECK-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
+; CHECK-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm2
; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
@@ -248,146 +215,58 @@ define void @test_maskz_store_v192i8(ptr %p0, <192 x i1> %mask) nounwind {
; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm1
-; CHECK-NEXT: vpbroadcastb {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k1
-; CHECK-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; CHECK-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k2
-; CHECK-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; CHECK-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; CHECK-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; CHECK-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k3
+; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; CHECK-NEXT: vpbroadcastb {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vpmovqb %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
+; CHECK-NEXT: vpmovqb %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vpmovqb %zmm4, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
+; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK-NEXT: vpmovqb %zmm0, %xmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vpmovqb %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
+; CHECK-NEXT: vpmovqb %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqb %zmm2, %xmm2
+; CHECK-NEXT: vpmovqb %zmm4, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
+; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k3
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovdqu8 %zmm0, 64(%rdi) {%k3}
; CHECK-NEXT: vmovdqu8 %zmm0, 128(%rdi) {%k2}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-mov.ll b/llvm/test/CodeGen/X86/avx512fp16-mov.ll
index 5e6c11e7fa100..79fc699f09932 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-mov.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-mov.ll
@@ -2138,21 +2138,9 @@ define <8 x half> @build_vector_xxxxxxxx(half %a0, half %a1, half %a2, half %a3,
;
; X86-LABEL: build_vector_xxxxxxxx:
; X86: # %bb.0:
-; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT: vpmovdw %ymm0, %xmm0
+; X86-NEXT: vzeroupper
; X86-NEXT: retl
%a = insertelement <8 x half> undef, half %a0, i32 0
%b = insertelement <8 x half> %a, half %a1, i32 1
@@ -2180,22 +2168,9 @@ define <16 x half> @build_vector_xxxxuuuuuuuuxxxx(half %a0, half %a1, half %a2,
;
; X86-LABEL: build_vector_xxxxuuuuuuuuxxxx:
; X86: # %bb.0:
-; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero
-; X86-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
-; X86-NEXT: vpbroadcastq %xmm0, %xmm0
-; X86-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT: vpmovdw %ymm0, %xmm0
+; X86-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; X86-NEXT: retl
%a = insertelement <16 x half> undef, half %a0, i32 0
%b = insertelement <16 x half> %a, half %a1, i32 1
diff --git a/llvm/test/CodeGen/X86/build-vector-128.ll b/llvm/test/CodeGen/X86/build-vector-128.ll
index 51d3a65be5236..e478e7037463a 100644
--- a/llvm/test/CodeGen/X86/build-vector-128.ll
+++ b/llvm/test/CodeGen/X86/build-vector-128.ll
@@ -207,17 +207,17 @@ define <8 x i16> @test_buildvector_v8i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16
; SSE41-64-NEXT: pinsrw $7, {{[0-9]+}}(%rsp), %xmm0
; SSE41-64-NEXT: retq
;
-; AVX-32-LABEL: test_buildvector_v8i16:
-; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: retl
+; AVX1-32-LABEL: test_buildvector_v8i16:
+; AVX1-32: # %bb.0:
+; AVX1-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v8i16:
; AVX-64: # %bb.0:
@@ -230,6 +230,15 @@ define <8 x i16> @test_buildvector_v8i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16
; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: retq
+;
+; AVX2-32-LABEL: test_buildvector_v8i16:
+; AVX2-32: # %bb.0:
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm0
+; AVX2-32-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-32-NEXT: vzeroupper
+; AVX2-32-NEXT: retl
%ins0 = insertelement <8 x i16> undef, i16 %a0, i32 0
%ins1 = insertelement <8 x i16> %ins0, i16 %a1, i32 1
%ins2 = insertelement <8 x i16> %ins1, i16 %a2, i32 2
@@ -572,17 +581,17 @@ define <16 x i8> @test_buildvector_v8i16_split_v16i8(i16 %a0, i16 %a1, i16 %a2,
; SSE41-64-NEXT: pinsrw $7, {{[0-9]+}}(%rsp), %xmm0
; SSE41-64-NEXT: retq
;
-; AVX-32-LABEL: test_buildvector_v8i16_split_v16i8:
-; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: retl
+; AVX1-32-LABEL: test_buildvector_v8i16_split_v16i8:
+; AVX1-32: # %bb.0:
+; AVX1-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v8i16_split_v16i8:
; AVX-64: # %bb.0:
@@ -595,6 +604,15 @@ define <16 x i8> @test_buildvector_v8i16_split_v16i8(i16 %a0, i16 %a1, i16 %a2,
; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: retq
+;
+; AVX2-32-LABEL: test_buildvector_v8i16_split_v16i8:
+; AVX2-32: # %bb.0:
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm0
+; AVX2-32-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-32-NEXT: vzeroupper
+; AVX2-32-NEXT: retl
%a0.lo = trunc i16 %a0 to i8
%a1.lo = trunc i16 %a1 to i8
%a2.lo = trunc i16 %a2 to i8
diff --git a/llvm/test/CodeGen/X86/build-vector-256.ll b/llvm/test/CodeGen/X86/build-vector-256.ll
index 773eb8f6742e5..2de346aee5857 100644
--- a/llvm/test/CodeGen/X86/build-vector-256.ll
+++ b/llvm/test/CodeGen/X86/build-vector-256.ll
@@ -171,23 +171,14 @@ define <16 x i16> @test_buildvector_v16i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
;
; AVX2-32-LABEL: test_buildvector_v16i16:
; AVX2-32: # %bb.0:
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm2
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm2, %ymm2
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
; AVX2-32-NEXT: retl
;
; AVX2-64-LABEL: test_buildvector_v16i16:
diff --git a/llvm/test/CodeGen/X86/build-vector-512.ll b/llvm/test/CodeGen/X86/build-vector-512.ll
index ff54288cfa5e0..ad4f76347f1c2 100644
--- a/llvm/test/CodeGen/X86/build-vector-512.ll
+++ b/llvm/test/CodeGen/X86/build-vector-512.ll
@@ -46,15 +46,8 @@ define <16 x float> @test_buildvector_v16f32(float %a0, float %a1, float %a2, fl
; AVX-64-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; AVX-64-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
; AVX-64-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; AVX-64-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
-; AVX-64-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; AVX-64-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; AVX-64-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX-64-NEXT: vpmovqd %zmm1, %ymm1
; AVX-64-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; AVX-64-NEXT: retq
%ins0 = insertelement <16 x float> undef, float %a0, i32 0
@@ -125,15 +118,8 @@ define <16 x i32> @test_buildvector_v16i32(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i
; AVX-64-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; AVX-64-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; AVX-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX-64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX-64-NEXT: vpmovqd %zmm1, %ymm1
; AVX-64-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX-64-NEXT: retq
%ins0 = insertelement <16 x i32> undef, i32 %a0, i32 0
@@ -158,61 +144,19 @@ define <16 x i32> @test_buildvector_v16i32(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i
define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15, i16 %a16, i16 %a17, i16 %a18, i16 %a19, i16 %a20, i16 %a21, i16 %a22, i16 %a23, i16 %a24, i16 %a25, i16 %a26, i16 %a27, i16 %a28, i16 %a29, i16 %a30, i16 %a31) {
; AVX-32-LABEL: test_buildvector_v32i16:
; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrw $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX-32-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm0
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm1
+; AVX-32-NEXT: vpmovdw %zmm0, %ymm0
+; AVX-32-NEXT: vpmovdw %zmm1, %ymm1
+; AVX-32-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v32i16:
; AVX-64: # %bb.0:
-; AVX-64-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrw $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX-64-NEXT: vpmovqw %zmm0, %xmm0
+; AVX-64-NEXT: vpmovqw %zmm1, %xmm1
; AVX-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX-64-NEXT: vmovd %edi, %xmm1
; AVX-64-NEXT: vpinsrw $1, %esi, %xmm1, %xmm1
@@ -222,14 +166,8 @@ define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
; AVX-64-NEXT: vpinsrw $5, %r9d, %xmm1, %xmm1
; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrw $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX-64-NEXT: vpmovqw %zmm2, %xmm2
; AVX-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX-64-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX-64-NEXT: retq
@@ -271,132 +209,43 @@ define <32 x i16> @test_buildvector_v32i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
define <64 x i8> @test_buildvector_v64i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15, i8 %a16, i8 %a17, i8 %a18, i8 %a19, i8 %a20, i8 %a21, i8 %a22, i8 %a23, i8 %a24, i8 %a25, i8 %a26, i8 %a27, i8 %a28, i8 %a29, i8 %a30, i8 %a31, i8 %a32, i8 %a33, i8 %a34, i8 %a35, i8 %a36, i8 %a37, i8 %a38, i8 %a39, i8 %a40, i8 %a41, i8 %a42, i8 %a43, i8 %a44, i8 %a45, i8 %a46, i8 %a47, i8 %a48, i8 %a49, i8 %a50, i8 %a51, i8 %a52, i8 %a53, i8 %a54, i8 %a55, i8 %a56, i8 %a57, i8 %a58, i8 %a59, i8 %a60, i8 %a61, i8 %a62, i8 %a63) {
; AVX-32-LABEL: test_buildvector_v64i8:
; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm0
+; AVX-32-NEXT: vpmovdb %zmm0, %xmm0
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm1
+; AVX-32-NEXT: vpmovdb %zmm1, %xmm1
; AVX-32-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm1
+; AVX-32-NEXT: vmovdqu64 {{[0-9]+}}(%esp), %zmm2
+; AVX-32-NEXT: vpmovdb %zmm1, %xmm1
+; AVX-32-NEXT: vpmovdb %zmm2, %xmm2
; AVX-32-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX-32-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v64i8:
; AVX-64: # %bb.0:
-; AVX-64-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX-64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX-64-NEXT: vmovd %edi, %xmm1
-; AVX-64-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX-64-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-64-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; AVX-64-NEXT: vpmovqb %zmm1, %xmm1
+; AVX-64-NEXT: vpmovqb %zmm3, %xmm3
+; AVX-64-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT: vpmovqb %zmm0, %xmm0
+; AVX-64-NEXT: vpmovqb %zmm2, %xmm2
+; AVX-64-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX-64-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX-64-NEXT: vpmovqb %zmm2, %xmm2
+; AVX-64-NEXT: vpmovqb %zmm1, %xmm1
+; AVX-64-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX-64-NEXT: vmovd %edi, %xmm2
+; AVX-64-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
+; AVX-64-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; AVX-64-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; AVX-64-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
+; AVX-64-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
; AVX-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
@@ -407,7 +256,7 @@ define <64 x i8> @test_buildvector_v64i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; AVX-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX-64-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX-64-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX-64-NEXT: retq
%ins0 = insertelement <64 x i8> undef, i8 %a0, i32 0
diff --git a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
new file mode 100644
index 0000000000000..e7f848a6e9ee9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
@@ -0,0 +1,1128 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512
+
+define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
+; AVX2-LABEL: buildvec_v16i8_stride8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 64(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i8_stride8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %p8 = getelementptr inbounds i8, ptr %p, i64 64
+ %p9 = getelementptr inbounds i8, ptr %p, i64 72
+ %p10 = getelementptr inbounds i8, ptr %p, i64 80
+ %p11 = getelementptr inbounds i8, ptr %p, i64 88
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <16 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <16 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <16 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <16 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <16 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <16 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <16 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <16 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <16 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
+; AVX2-LABEL: buildvec_v16i8_stride4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i8_stride4:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 4
+ %p2 = getelementptr inbounds i8, ptr %p, i64 8
+ %p3 = getelementptr inbounds i8, ptr %p, i64 12
+ %p4 = getelementptr inbounds i8, ptr %p, i64 16
+ %p5 = getelementptr inbounds i8, ptr %p, i64 20
+ %p6 = getelementptr inbounds i8, ptr %p, i64 24
+ %p7 = getelementptr inbounds i8, ptr %p, i64 28
+ %p8 = getelementptr inbounds i8, ptr %p, i64 32
+ %p9 = getelementptr inbounds i8, ptr %p, i64 36
+ %p10 = getelementptr inbounds i8, ptr %p, i64 40
+ %p11 = getelementptr inbounds i8, ptr %p, i64 44
+ %p12 = getelementptr inbounds i8, ptr %p, i64 48
+ %p13 = getelementptr inbounds i8, ptr %p, i64 52
+ %p14 = getelementptr inbounds i8, ptr %p, i64 56
+ %p15 = getelementptr inbounds i8, ptr %p, i64 60
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <16 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <16 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <16 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <16 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <16 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <16 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <16 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <16 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <16 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <8 x i16> @buildvec_v8i16_stride4(ptr %p) {
+; AVX2-LABEL: buildvec_v8i16_stride4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqu (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v8i16_stride4:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu (%rdi), %ymm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 4
+ %p2 = getelementptr inbounds i8, ptr %p, i64 8
+ %p3 = getelementptr inbounds i8, ptr %p, i64 12
+ %p4 = getelementptr inbounds i8, ptr %p, i64 16
+ %p5 = getelementptr inbounds i8, ptr %p, i64 20
+ %p6 = getelementptr inbounds i8, ptr %p, i64 24
+ %p7 = getelementptr inbounds i8, ptr %p, i64 28
+ %v0 = load i16, ptr %p0
+ %v1 = load i16, ptr %p1
+ %v2 = load i16, ptr %p2
+ %v3 = load i16, ptr %p3
+ %v4 = load i16, ptr %p4
+ %v5 = load i16, ptr %p5
+ %v6 = load i16, ptr %p6
+ %v7 = load i16, ptr %p7
+ %r0 = insertelement <8 x i16> poison, i16 %v0, i32 0
+ %r1 = insertelement <8 x i16> %r0, i16 %v1, i32 1
+ %r2 = insertelement <8 x i16> %r1, i16 %v2, i32 2
+ %r3 = insertelement <8 x i16> %r2, i16 %v3, i32 3
+ %r4 = insertelement <8 x i16> %r3, i16 %v4, i32 4
+ %r5 = insertelement <8 x i16> %r4, i16 %v5, i32 5
+ %r6 = insertelement <8 x i16> %r5, i16 %v6, i32 6
+ %r7 = insertelement <8 x i16> %r6, i16 %v7, i32 7
+ ret <8 x i16> %r7
+}
+
+define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
+; AVX2-LABEL: buildvec_v16i16_stride8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzwl 64(%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrw $1, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $2, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $3, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $4, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $5, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $6, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $7, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: movzwl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpinsrw $1, 8(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $2, 16(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $3, 24(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $4, 32(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $5, 40(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $6, 48(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $7, 56(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i16_stride8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %p8 = getelementptr inbounds i8, ptr %p, i64 64
+ %p9 = getelementptr inbounds i8, ptr %p, i64 72
+ %p10 = getelementptr inbounds i8, ptr %p, i64 80
+ %p11 = getelementptr inbounds i8, ptr %p, i64 88
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %v0 = load i16, ptr %p0
+ %v1 = load i16, ptr %p1
+ %v2 = load i16, ptr %p2
+ %v3 = load i16, ptr %p3
+ %v4 = load i16, ptr %p4
+ %v5 = load i16, ptr %p5
+ %v6 = load i16, ptr %p6
+ %v7 = load i16, ptr %p7
+ %v8 = load i16, ptr %p8
+ %v9 = load i16, ptr %p9
+ %v10 = load i16, ptr %p10
+ %v11 = load i16, ptr %p11
+ %v12 = load i16, ptr %p12
+ %v13 = load i16, ptr %p13
+ %v14 = load i16, ptr %p14
+ %v15 = load i16, ptr %p15
+ %r0 = insertelement <16 x i16> poison, i16 %v0, i32 0
+ %r1 = insertelement <16 x i16> %r0, i16 %v1, i32 1
+ %r2 = insertelement <16 x i16> %r1, i16 %v2, i32 2
+ %r3 = insertelement <16 x i16> %r2, i16 %v3, i32 3
+ %r4 = insertelement <16 x i16> %r3, i16 %v4, i32 4
+ %r5 = insertelement <16 x i16> %r4, i16 %v5, i32 5
+ %r6 = insertelement <16 x i16> %r5, i16 %v6, i32 6
+ %r7 = insertelement <16 x i16> %r6, i16 %v7, i32 7
+ %r8 = insertelement <16 x i16> %r7, i16 %v8, i32 8
+ %r9 = insertelement <16 x i16> %r8, i16 %v9, i32 9
+ %r10 = insertelement <16 x i16> %r9, i16 %v10, i32 10
+ %r11 = insertelement <16 x i16> %r10, i16 %v11, i32 11
+ %r12 = insertelement <16 x i16> %r11, i16 %v12, i32 12
+ %r13 = insertelement <16 x i16> %r12, i16 %v13, i32 13
+ %r14 = insertelement <16 x i16> %r13, i16 %v14, i32 14
+ %r15 = insertelement <16 x i16> %r14, i16 %v15, i32 15
+ ret <16 x i16> %r15
+}
+
+define <32 x i8> @buildvec_v32i8_stride4(ptr %p) {
+; AVX2-LABEL: buildvec_v32i8_stride4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl 64(%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 68(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 76(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 84(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 92(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 100(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 108(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 116(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 124(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v32i8_stride4:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 4
+ %p2 = getelementptr inbounds i8, ptr %p, i64 8
+ %p3 = getelementptr inbounds i8, ptr %p, i64 12
+ %p4 = getelementptr inbounds i8, ptr %p, i64 16
+ %p5 = getelementptr inbounds i8, ptr %p, i64 20
+ %p6 = getelementptr inbounds i8, ptr %p, i64 24
+ %p7 = getelementptr inbounds i8, ptr %p, i64 28
+ %p8 = getelementptr inbounds i8, ptr %p, i64 32
+ %p9 = getelementptr inbounds i8, ptr %p, i64 36
+ %p10 = getelementptr inbounds i8, ptr %p, i64 40
+ %p11 = getelementptr inbounds i8, ptr %p, i64 44
+ %p12 = getelementptr inbounds i8, ptr %p, i64 48
+ %p13 = getelementptr inbounds i8, ptr %p, i64 52
+ %p14 = getelementptr inbounds i8, ptr %p, i64 56
+ %p15 = getelementptr inbounds i8, ptr %p, i64 60
+ %p16 = getelementptr inbounds i8, ptr %p, i64 64
+ %p17 = getelementptr inbounds i8, ptr %p, i64 68
+ %p18 = getelementptr inbounds i8, ptr %p, i64 72
+ %p19 = getelementptr inbounds i8, ptr %p, i64 76
+ %p20 = getelementptr inbounds i8, ptr %p, i64 80
+ %p21 = getelementptr inbounds i8, ptr %p, i64 84
+ %p22 = getelementptr inbounds i8, ptr %p, i64 88
+ %p23 = getelementptr inbounds i8, ptr %p, i64 92
+ %p24 = getelementptr inbounds i8, ptr %p, i64 96
+ %p25 = getelementptr inbounds i8, ptr %p, i64 100
+ %p26 = getelementptr inbounds i8, ptr %p, i64 104
+ %p27 = getelementptr inbounds i8, ptr %p, i64 108
+ %p28 = getelementptr inbounds i8, ptr %p, i64 112
+ %p29 = getelementptr inbounds i8, ptr %p, i64 116
+ %p30 = getelementptr inbounds i8, ptr %p, i64 120
+ %p31 = getelementptr inbounds i8, ptr %p, i64 124
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %v16 = load i8, ptr %p16
+ %v17 = load i8, ptr %p17
+ %v18 = load i8, ptr %p18
+ %v19 = load i8, ptr %p19
+ %v20 = load i8, ptr %p20
+ %v21 = load i8, ptr %p21
+ %v22 = load i8, ptr %p22
+ %v23 = load i8, ptr %p23
+ %v24 = load i8, ptr %p24
+ %v25 = load i8, ptr %p25
+ %v26 = load i8, ptr %p26
+ %v27 = load i8, ptr %p27
+ %v28 = load i8, ptr %p28
+ %v29 = load i8, ptr %p29
+ %v30 = load i8, ptr %p30
+ %v31 = load i8, ptr %p31
+ %r0 = insertelement <32 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <32 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <32 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <32 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <32 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <32 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <32 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <32 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <32 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <32 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <32 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <32 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <32 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <32 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <32 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <32 x i8> %r14, i8 %v15, i32 15
+ %r16 = insertelement <32 x i8> %r15, i8 %v16, i32 16
+ %r17 = insertelement <32 x i8> %r16, i8 %v17, i32 17
+ %r18 = insertelement <32 x i8> %r17, i8 %v18, i32 18
+ %r19 = insertelement <32 x i8> %r18, i8 %v19, i32 19
+ %r20 = insertelement <32 x i8> %r19, i8 %v20, i32 20
+ %r21 = insertelement <32 x i8> %r20, i8 %v21, i32 21
+ %r22 = insertelement <32 x i8> %r21, i8 %v22, i32 22
+ %r23 = insertelement <32 x i8> %r22, i8 %v23, i32 23
+ %r24 = insertelement <32 x i8> %r23, i8 %v24, i32 24
+ %r25 = insertelement <32 x i8> %r24, i8 %v25, i32 25
+ %r26 = insertelement <32 x i8> %r25, i8 %v26, i32 26
+ %r27 = insertelement <32 x i8> %r26, i8 %v27, i32 27
+ %r28 = insertelement <32 x i8> %r27, i8 %v28, i32 28
+ %r29 = insertelement <32 x i8> %r28, i8 %v29, i32 29
+ %r30 = insertelement <32 x i8> %r29, i8 %v30, i32 30
+ %r31 = insertelement <32 x i8> %r30, i8 %v31, i32 31
+ ret <32 x i8> %r31
+}
+
+define <16 x i8> @buildvec_v16i8_stride8_volatile(ptr %p) {
+; AVX2-LABEL: buildvec_v16i8_stride8_volatile:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 64(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i8_stride8_volatile:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $8, 64(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $9, 72(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $10, 80(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $11, 88(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %p8 = getelementptr inbounds i8, ptr %p, i64 64
+ %p9 = getelementptr inbounds i8, ptr %p, i64 72
+ %p10 = getelementptr inbounds i8, ptr %p, i64 80
+ %p11 = getelementptr inbounds i8, ptr %p, i64 88
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %v0 = load volatile i8, ptr %p0
+ %v1 = load volatile i8, ptr %p1
+ %v2 = load volatile i8, ptr %p2
+ %v3 = load volatile i8, ptr %p3
+ %v4 = load volatile i8, ptr %p4
+ %v5 = load volatile i8, ptr %p5
+ %v6 = load volatile i8, ptr %p6
+ %v7 = load volatile i8, ptr %p7
+ %v8 = load volatile i8, ptr %p8
+ %v9 = load volatile i8, ptr %p9
+ %v10 = load volatile i8, ptr %p10
+ %v11 = load volatile i8, ptr %p11
+ %v12 = load volatile i8, ptr %p12
+ %v13 = load volatile i8, ptr %p13
+ %v14 = load volatile i8, ptr %p14
+ %v15 = load volatile i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <16 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <16 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <16 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <16 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <16 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <16 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <16 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <16 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <16 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <16 x i8> @buildvec_v16i8_stride8_one_skewed(ptr %p) {
+; AVX2-LABEL: buildvec_v16i8_stride8_one_skewed:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 65(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i8_stride8_one_skewed:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $8, 65(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $9, 72(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $10, 80(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $11, 88(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %p8 = getelementptr inbounds i8, ptr %p, i64 65 ; not aligned with the stride
+ %p9 = getelementptr inbounds i8, ptr %p, i64 72
+ %p10 = getelementptr inbounds i8, ptr %p, i64 80
+ %p11 = getelementptr inbounds i8, ptr %p, i64 88
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <16 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <16 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <16 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <16 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <16 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <16 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <16 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <16 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <16 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <32 x i8> @buildvec_v32i8_stride8(ptr %p) {
+; AVX2-LABEL: buildvec_v32i8_stride8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl 128(%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 136(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 144(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 152(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 160(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 168(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 176(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 184(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 192(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 200(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 208(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 216(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 224(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 232(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 240(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 248(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $8, 64(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $9, 72(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $10, 80(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $11, 88(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $12, 96(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $13, 104(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $14, 112(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v32i8_stride8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512-NEXT: vmovdqu64 128(%rdi), %zmm2
+; AVX512-NEXT: vmovdqu64 192(%rdi), %zmm3
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %p8 = getelementptr inbounds i8, ptr %p, i64 64
+ %p9 = getelementptr inbounds i8, ptr %p, i64 72
+ %p10 = getelementptr inbounds i8, ptr %p, i64 80
+ %p11 = getelementptr inbounds i8, ptr %p, i64 88
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %p16 = getelementptr inbounds i8, ptr %p, i64 128
+ %p17 = getelementptr inbounds i8, ptr %p, i64 136
+ %p18 = getelementptr inbounds i8, ptr %p, i64 144
+ %p19 = getelementptr inbounds i8, ptr %p, i64 152
+ %p20 = getelementptr inbounds i8, ptr %p, i64 160
+ %p21 = getelementptr inbounds i8, ptr %p, i64 168
+ %p22 = getelementptr inbounds i8, ptr %p, i64 176
+ %p23 = getelementptr inbounds i8, ptr %p, i64 184
+ %p24 = getelementptr inbounds i8, ptr %p, i64 192
+ %p25 = getelementptr inbounds i8, ptr %p, i64 200
+ %p26 = getelementptr inbounds i8, ptr %p, i64 208
+ %p27 = getelementptr inbounds i8, ptr %p, i64 216
+ %p28 = getelementptr inbounds i8, ptr %p, i64 224
+ %p29 = getelementptr inbounds i8, ptr %p, i64 232
+ %p30 = getelementptr inbounds i8, ptr %p, i64 240
+ %p31 = getelementptr inbounds i8, ptr %p, i64 248
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %v16 = load i8, ptr %p16
+ %v17 = load i8, ptr %p17
+ %v18 = load i8, ptr %p18
+ %v19 = load i8, ptr %p19
+ %v20 = load i8, ptr %p20
+ %v21 = load i8, ptr %p21
+ %v22 = load i8, ptr %p22
+ %v23 = load i8, ptr %p23
+ %v24 = load i8, ptr %p24
+ %v25 = load i8, ptr %p25
+ %v26 = load i8, ptr %p26
+ %v27 = load i8, ptr %p27
+ %v28 = load i8, ptr %p28
+ %v29 = load i8, ptr %p29
+ %v30 = load i8, ptr %p30
+ %v31 = load i8, ptr %p31
+ %r0 = insertelement <32 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <32 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <32 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <32 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <32 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <32 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <32 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <32 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <32 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <32 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <32 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <32 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <32 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <32 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <32 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <32 x i8> %r14, i8 %v15, i32 15
+ %r16 = insertelement <32 x i8> %r15, i8 %v16, i32 16
+ %r17 = insertelement <32 x i8> %r16, i8 %v17, i32 17
+ %r18 = insertelement <32 x i8> %r17, i8 %v18, i32 18
+ %r19 = insertelement <32 x i8> %r18, i8 %v19, i32 19
+ %r20 = insertelement <32 x i8> %r19, i8 %v20, i32 20
+ %r21 = insertelement <32 x i8> %r20, i8 %v21, i32 21
+ %r22 = insertelement <32 x i8> %r21, i8 %v22, i32 22
+ %r23 = insertelement <32 x i8> %r22, i8 %v23, i32 23
+ %r24 = insertelement <32 x i8> %r23, i8 %v24, i32 24
+ %r25 = insertelement <32 x i8> %r24, i8 %v25, i32 25
+ %r26 = insertelement <32 x i8> %r25, i8 %v26, i32 26
+ %r27 = insertelement <32 x i8> %r26, i8 %v27, i32 27
+ %r28 = insertelement <32 x i8> %r27, i8 %v28, i32 28
+ %r29 = insertelement <32 x i8> %r28, i8 %v29, i32 29
+ %r30 = insertelement <32 x i8> %r29, i8 %v30, i32 30
+ %r31 = insertelement <32 x i8> %r30, i8 %v31, i32 31
+ ret <32 x i8> %r31
+}
+
+define <8 x i8> @buildvec_v8i8_stride8(ptr %p) {
+; AVX2-LABEL: buildvec_v8i8_stride8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v8i8_stride8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %r0 = insertelement <8 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <8 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <8 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <8 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <8 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <8 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <8 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <8 x i8> %r6, i8 %v7, i32 7
+ ret <8 x i8> %r7
+}
+
+define <16 x i8> @buildvec_interleaved_undef(ptr %p) {
+; AVX2-LABEL: buildvec_interleaved_undef:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_interleaved_undef:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $12, 96(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $13, 104(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $14, 112(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p12 = getelementptr inbounds i8, ptr %p, i64 96
+ %p13 = getelementptr inbounds i8, ptr %p, i64 104
+ %p14 = getelementptr inbounds i8, ptr %p, i64 112
+ %p15 = getelementptr inbounds i8, ptr %p, i64 120
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r12 = insertelement <16 x i8> %r3, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <32 x i8> @buildvec_v32i8_stride8_basepos16(ptr %p) {
+; AVX2-LABEL: buildvec_v32i8_stride8_basepos16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v32i8_stride8_basepos16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %r0 = insertelement <32 x i8> poison, i8 %v0, i32 16
+ %r1 = insertelement <32 x i8> %r0, i8 %v1, i32 17
+ %r2 = insertelement <32 x i8> %r1, i8 %v2, i32 18
+ %r3 = insertelement <32 x i8> %r2, i8 %v3, i32 19
+ %r4 = insertelement <32 x i8> %r3, i8 %v4, i32 20
+ %r5 = insertelement <32 x i8> %r4, i8 %v5, i32 21
+ %r6 = insertelement <32 x i8> %r5, i8 %v6, i32 22
+ %r7 = insertelement <32 x i8> %r6, i8 %v7, i32 23
+ ret <32 x i8> %r7
+}
+
+define <32 x i8> @buildvec_v32i8_stride8_basepos8(ptr %p) {
+; AVX2-LABEL: buildvec_v32i8_stride8_basepos8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, (%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v32i8_stride8_basepos8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $8, (%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $9, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $10, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $11, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $12, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $13, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $14, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $15, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %r0 = insertelement <32 x i8> poison, i8 %v0, i32 8
+ %r1 = insertelement <32 x i8> %r0, i8 %v1, i32 9
+ %r2 = insertelement <32 x i8> %r1, i8 %v2, i32 10
+ %r3 = insertelement <32 x i8> %r2, i8 %v3, i32 11
+ %r4 = insertelement <32 x i8> %r3, i8 %v4, i32 12
+ %r5 = insertelement <32 x i8> %r4, i8 %v5, i32 13
+ %r6 = insertelement <32 x i8> %r5, i8 %v6, i32 14
+ %r7 = insertelement <32 x i8> %r6, i8 %v7, i32 15
+ ret <32 x i8> %r7
+}
+
+define <16 x i8> @buildvec_v16i8_stride8_reverse(ptr %p) {
+; AVX2-LABEL: buildvec_v16i8_stride8_reverse:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl 120(%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 112(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 104(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 96(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 88(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 80(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 72(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 64(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $8, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $9, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $11, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $12, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $13, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $14, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $15, (%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v16i8_stride8_reverse:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 120
+ %p1 = getelementptr inbounds i8, ptr %p, i64 112
+ %p2 = getelementptr inbounds i8, ptr %p, i64 104
+ %p3 = getelementptr inbounds i8, ptr %p, i64 96
+ %p4 = getelementptr inbounds i8, ptr %p, i64 88
+ %p5 = getelementptr inbounds i8, ptr %p, i64 80
+ %p6 = getelementptr inbounds i8, ptr %p, i64 72
+ %p7 = getelementptr inbounds i8, ptr %p, i64 64
+ %p8 = getelementptr inbounds i8, ptr %p, i64 56
+ %p9 = getelementptr inbounds i8, ptr %p, i64 48
+ %p10 = getelementptr inbounds i8, ptr %p, i64 40
+ %p11 = getelementptr inbounds i8, ptr %p, i64 32
+ %p12 = getelementptr inbounds i8, ptr %p, i64 24
+ %p13 = getelementptr inbounds i8, ptr %p, i64 16
+ %p14 = getelementptr inbounds i8, ptr %p, i64 8
+ %p15 = getelementptr inbounds i8, ptr %p, i64 0
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %v8 = load i8, ptr %p8
+ %v9 = load i8, ptr %p9
+ %v10 = load i8, ptr %p10
+ %v11 = load i8, ptr %p11
+ %v12 = load i8, ptr %p12
+ %v13 = load i8, ptr %p13
+ %v14 = load i8, ptr %p14
+ %v15 = load i8, ptr %p15
+ %r0 = insertelement <16 x i8> poison, i8 %v0, i32 0
+ %r1 = insertelement <16 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <16 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <16 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <16 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <16 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <16 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <16 x i8> %r6, i8 %v7, i32 7
+ %r8 = insertelement <16 x i8> %r7, i8 %v8, i32 8
+ %r9 = insertelement <16 x i8> %r8, i8 %v9, i32 9
+ %r10 = insertelement <16 x i8> %r9, i8 %v10, i32 10
+ %r11 = insertelement <16 x i8> %r10, i8 %v11, i32 11
+ %r12 = insertelement <16 x i8> %r11, i8 %v12, i32 12
+ %r13 = insertelement <16 x i8> %r12, i8 %v13, i32 13
+ %r14 = insertelement <16 x i8> %r13, i8 %v14, i32 14
+ %r15 = insertelement <16 x i8> %r14, i8 %v15, i32 15
+ ret <16 x i8> %r15
+}
+
+define <32 x i8> @buildvec_v32i8_stride8_trailing_zeros(ptr %p) {
+; AVX2-LABEL: buildvec_v32i8_stride8_trailing_zeros:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: buildvec_v32i8_stride8_trailing_zeros:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movzbl (%rdi), %eax
+; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrb $1, 8(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, 16(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, 24(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, 32(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, 40(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, 48(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, 56(%rdi), %xmm0, %xmm0
+; AVX512-NEXT: retq
+ %p0 = getelementptr inbounds i8, ptr %p, i64 0
+ %p1 = getelementptr inbounds i8, ptr %p, i64 8
+ %p2 = getelementptr inbounds i8, ptr %p, i64 16
+ %p3 = getelementptr inbounds i8, ptr %p, i64 24
+ %p4 = getelementptr inbounds i8, ptr %p, i64 32
+ %p5 = getelementptr inbounds i8, ptr %p, i64 40
+ %p6 = getelementptr inbounds i8, ptr %p, i64 48
+ %p7 = getelementptr inbounds i8, ptr %p, i64 56
+ %v0 = load i8, ptr %p0
+ %v1 = load i8, ptr %p1
+ %v2 = load i8, ptr %p2
+ %v3 = load i8, ptr %p3
+ %v4 = load i8, ptr %p4
+ %v5 = load i8, ptr %p5
+ %v6 = load i8, ptr %p6
+ %v7 = load i8, ptr %p7
+ %r0 = insertelement <32 x i8> zeroinitializer, i8 %v0, i32 0
+ %r1 = insertelement <32 x i8> %r0, i8 %v1, i32 1
+ %r2 = insertelement <32 x i8> %r1, i8 %v2, i32 2
+ %r3 = insertelement <32 x i8> %r2, i8 %v3, i32 3
+ %r4 = insertelement <32 x i8> %r3, i8 %v4, i32 4
+ %r5 = insertelement <32 x i8> %r4, i8 %v5, i32 5
+ %r6 = insertelement <32 x i8> %r5, i8 %v6, i32 6
+ %r7 = insertelement <32 x i8> %r6, i8 %v7, i32 7
+ ret <32 x i8> %r7
+}
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
index 82f32898244a6..66bd427f43cd4 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
@@ -459,42 +459,28 @@ define void @test_mscatter_v17f32(ptr %base, <17 x i32> %index, <17 x float> %va
; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
; WIDEN_SKX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
-; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; WIDEN_SKX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; WIDEN_SKX-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; WIDEN_SKX-NEXT: vpmovqd %zmm1, %ymm1
; WIDEN_SKX-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; WIDEN_SKX-NEXT: vmovd %esi, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $1, %edx, %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $3, %r8d, %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vmovd %r9d, %xmm3
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_SKX-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; WIDEN_SKX-NEXT: vmovd %esi, %xmm3
+; WIDEN_SKX-NEXT: vpinsrd $1, %edx, %xmm3, %xmm3
+; WIDEN_SKX-NEXT: vpinsrd $2, %ecx, %xmm3, %xmm3
+; WIDEN_SKX-NEXT: vpinsrd $3, %r8d, %xmm3, %xmm3
+; WIDEN_SKX-NEXT: vmovd %r9d, %xmm4
+; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; WIDEN_SKX-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; WIDEN_SKX-NEXT: vpmovqd %zmm4, %ymm4
+; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3
; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1
-; WIDEN_SKX-NEXT: vscatterdps %zmm0, (%rdi,%zmm1,4) {%k1}
+; WIDEN_SKX-NEXT: vscatterdps %zmm0, (%rdi,%zmm3,4) {%k1}
; WIDEN_SKX-NEXT: movw $1, %ax
; WIDEN_SKX-NEXT: kmovw %eax, %k1
-; WIDEN_SKX-NEXT: vscatterdps %zmm2, (%rdi,%zmm3,4) {%k1}
+; WIDEN_SKX-NEXT: vscatterdps %zmm1, (%rdi,%zmm2,4) {%k1}
; WIDEN_SKX-NEXT: vzeroupper
; WIDEN_SKX-NEXT: retq
;
@@ -507,42 +493,28 @@ define void @test_mscatter_v17f32(ptr %base, <17 x i32> %index, <17 x float> %va
; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]
; WIDEN_KNL-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0
-; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]
-; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
-; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
-; WIDEN_KNL-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; WIDEN_KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; WIDEN_KNL-NEXT: vpmovqd %zmm1, %ymm1
; WIDEN_KNL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; WIDEN_KNL-NEXT: vmovd %esi, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $1, %edx, %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $3, %r8d, %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vmovd %r9d, %xmm3
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
-; WIDEN_KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; WIDEN_KNL-NEXT: vmovd %esi, %xmm3
+; WIDEN_KNL-NEXT: vpinsrd $1, %edx, %xmm3, %xmm3
+; WIDEN_KNL-NEXT: vpinsrd $2, %ecx, %xmm3, %xmm3
+; WIDEN_KNL-NEXT: vpinsrd $3, %r8d, %xmm3, %xmm3
+; WIDEN_KNL-NEXT: vmovd %r9d, %xmm4
+; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4
+; WIDEN_KNL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; WIDEN_KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; WIDEN_KNL-NEXT: vpmovqd %zmm4, %ymm4
+; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3
; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1
-; WIDEN_KNL-NEXT: vscatterdps %zmm0, (%rdi,%zmm1,4) {%k1}
+; WIDEN_KNL-NEXT: vscatterdps %zmm0, (%rdi,%zmm3,4) {%k1}
; WIDEN_KNL-NEXT: movw $1, %ax
; WIDEN_KNL-NEXT: kmovw %eax, %k1
-; WIDEN_KNL-NEXT: vscatterdps %zmm2, (%rdi,%zmm3,4) {%k1}
+; WIDEN_KNL-NEXT: vscatterdps %zmm1, (%rdi,%zmm2,4) {%k1}
; WIDEN_KNL-NEXT: vzeroupper
; WIDEN_KNL-NEXT: retq
;
@@ -581,47 +553,42 @@ define void @test_mscatter_v17f32(ptr %base, <17 x i32> %index, <17 x float> %va
; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax
; WIDEN_AVX2-NEXT: vmovss %xmm6, (%rax)
; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_AVX2-NEXT: vpmovsxdq %xmm0, %ymm0
-; WIDEN_AVX2-NEXT: vpsllq $2, %ymm0, %ymm0
-; WIDEN_AVX2-NEXT: vpaddq %ymm0, %ymm8, %ymm0
+; WIDEN_AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,2,4,6,4,6,6,7]
; WIDEN_AVX2-NEXT: vmovss %xmm7, (%rax)
-; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm1
; WIDEN_AVX2-NEXT: vpmovsxdq %xmm1, %ymm1
; WIDEN_AVX2-NEXT: vpsllq $2, %ymm1, %ymm1
; WIDEN_AVX2-NEXT: vpaddq %ymm1, %ymm8, %ymm1
+; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm0
+; WIDEN_AVX2-NEXT: vpmovsxdq %xmm0, %ymm0
+; WIDEN_AVX2-NEXT: vpsllq $2, %ymm0, %ymm0
+; WIDEN_AVX2-NEXT: vpaddq %ymm0, %ymm8, %ymm0
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax
; WIDEN_AVX2-NEXT: vmovss %xmm2, (%rax)
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax
; WIDEN_AVX2-NEXT: vmovss %xmm2, (%rax)
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
-; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax
-; WIDEN_AVX2-NEXT: vmovss %xmm2, (%rax)
-; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)
-; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax
-; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)
-; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax
-; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)
-; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax
-; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)
+; WIDEN_AVX2-NEXT: vmovss %xmm2, (%rax)
; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)
; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; WIDEN_AVX2-NEXT: vpmovsxdq %xmm0, %xmm0
; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax
@@ -639,63 +606,45 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
; WIDEN_SKX-LABEL: test_mgather_v17f32:
; WIDEN_SKX: # %bb.0:
; WIDEN_SKX-NEXT: movq %rdi, %rax
-; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; WIDEN_SKX-NEXT: vmovd %edx, %xmm0
+; WIDEN_SKX-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
+; WIDEN_SKX-NEXT: vpinsrd $2, %r8d, %xmm0, %xmm0
+; WIDEN_SKX-NEXT: vpinsrd $3, %r9d, %xmm0, %xmm0
+; WIDEN_SKX-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm1
+; WIDEN_SKX-NEXT: vpmovqd %ymm1, %xmm1
; WIDEN_SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; WIDEN_SKX-NEXT: vmovd %edx, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $2, %r8d, %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vpinsrd $3, %r9d, %xmm1, %xmm1
-; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; WIDEN_SKX-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; WIDEN_SKX-NEXT: vpmovqd %zmm1, %ymm1
+; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1
-; WIDEN_SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; WIDEN_SKX-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}
+; WIDEN_SKX-NEXT: vxorps %xmm2, %xmm2, %xmm2
; WIDEN_SKX-NEXT: movw $1, %cx
; WIDEN_SKX-NEXT: kmovw %ecx, %k1
-; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm1,4), %zmm2 {%k1}
-; WIDEN_SKX-NEXT: vmovss %xmm2, 64(%rdi)
-; WIDEN_SKX-NEXT: vmovaps %zmm3, (%rdi)
+; WIDEN_SKX-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm1,4), %zmm3 {%k1}
+; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1
+; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm2 {%k1}
+; WIDEN_SKX-NEXT: vmovss %xmm3, 64(%rdi)
+; WIDEN_SKX-NEXT: vmovaps %zmm2, (%rdi)
; WIDEN_SKX-NEXT: vzeroupper
; WIDEN_SKX-NEXT: retq
;
; WIDEN_KNL-LABEL: test_mgather_v17f32:
; WIDEN_KNL: # %bb.0:
; WIDEN_KNL-NEXT: movq %rdi, %rax
-; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; WIDEN_KNL-NEXT: vmovd %edx, %xmm0
+; WIDEN_KNL-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
+; WIDEN_KNL-NEXT: vpinsrd $2, %r8d, %xmm0, %xmm0
+; WIDEN_KNL-NEXT: vpinsrd $3, %r9d, %xmm0, %xmm0
+; WIDEN_KNL-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm1
+; WIDEN_KNL-NEXT: vpmovqd %zmm1, %ymm1
; WIDEN_KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; WIDEN_KNL-NEXT: vmovd %edx, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $2, %r8d, %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vpinsrd $3, %r9d, %xmm1, %xmm1
-; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; WIDEN_KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; WIDEN_KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; WIDEN_KNL-NEXT: vpmovqd %zmm1, %ymm1
+; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1
-; WIDEN_KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; WIDEN_KNL-NEXT: vxorps %xmm2, %xmm2, %xmm2
; WIDEN_KNL-NEXT: vxorps %xmm3, %xmm3, %xmm3
; WIDEN_KNL-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}
; WIDEN_KNL-NEXT: movw $1, %cx
@@ -708,38 +657,31 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
;
; WIDEN_AVX2-LABEL: test_mgather_v17f32:
; WIDEN_AVX2: # %bb.0:
-; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; WIDEN_AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
+; WIDEN_AVX2-NEXT: # ymm0 = mem[0,1,0,1]
+; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm1
; WIDEN_AVX2-NEXT: movq %rdi, %rax
-; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm2
+; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; WIDEN_AVX2-NEXT: vmovd %edx, %xmm3
; WIDEN_AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3
; WIDEN_AVX2-NEXT: vpinsrd $2, %r8d, %xmm3, %xmm3
; WIDEN_AVX2-NEXT: vpinsrd $3, %r9d, %xmm3, %xmm3
-; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm0
+; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
; WIDEN_AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
-; WIDEN_AVX2-NEXT: vxorps %xmm4, %xmm4, %xmm4
-; WIDEN_AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
-; WIDEN_AVX2-NEXT: vxorps %xmm6, %xmm6, %xmm6
-; WIDEN_AVX2-NEXT: vgatherdps %ymm5, (%rsi,%ymm1,4), %ymm6
-; WIDEN_AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vgatherdps %ymm3, (%rsi,%ymm0,4), %ymm1
+; WIDEN_AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
+; WIDEN_AVX2-NEXT: vxorps %xmm5, %xmm5, %xmm5
+; WIDEN_AVX2-NEXT: vgatherdps %ymm4, (%rsi,%ymm0,4), %ymm5
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = [4294967295,0,0,0]
+; WIDEN_AVX2-NEXT: vxorps %xmm4, %xmm4, %xmm4
; WIDEN_AVX2-NEXT: vgatherdps %ymm0, (%rsi,%ymm2,4), %ymm4
+; WIDEN_AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; WIDEN_AVX2-NEXT: vgatherdps %ymm3, (%rsi,%ymm1,4), %ymm0
+; WIDEN_AVX2-NEXT: vmovaps %ymm0, 32(%rdi)
; WIDEN_AVX2-NEXT: vmovss %xmm4, 64(%rdi)
-; WIDEN_AVX2-NEXT: vmovaps %ymm1, 32(%rdi)
-; WIDEN_AVX2-NEXT: vmovaps %ymm6, (%rdi)
+; WIDEN_AVX2-NEXT: vmovaps %ymm5, (%rdi)
; WIDEN_AVX2-NEXT: vzeroupper
; WIDEN_AVX2-NEXT: retq
{
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 018c3d36ee60d..3b58bbd719d79 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -2662,104 +2662,71 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: movq %rsp, %rbp
; AVX512F-NEXT: andq $-64, %rsp
; AVX512F-NEXT: subq $192, %rsp
-; AVX512F-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 360(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $2, 368(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $3, 376(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $4, 384(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $5, 392(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $6, 400(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $7, 408(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $8, 416(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $9, 424(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $10, 432(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $11, 440(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $12, 448(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $13, 456(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $14, 464(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $15, 472(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vptestmd %zmm2, %zmm3, %k1
-; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm4, %xmm3
-; AVX512F-NEXT: vmovd %edi, %xmm4
-; AVX512F-NEXT: vpinsrb $1, %esi, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, %edx, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm4, %xmm4
+; AVX512F-NEXT: vmovdqu64 352(%rbp), %zmm2
+; AVX512F-NEXT: vmovdqu64 416(%rbp), %zmm3
+; AVX512F-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vmovdqu64 224(%rbp), %zmm2
+; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm4
+; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm2[0],xmm4[0]
+; AVX512F-NEXT: vmovdqu64 96(%rbp), %zmm2
+; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm6
+; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
+; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; AVX512F-NEXT: vmovd %edi, %xmm6
+; AVX512F-NEXT: vpinsrb $1, %esi, %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $2, %edx, %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k2
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
+; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm6
+; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm8
+; AVX512F-NEXT: vpaddd %ymm6, %ymm8, %ymm6
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm8
+; AVX512F-NEXT: vpaddd %xmm6, %xmm8, %xmm6
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm6, %xmm8, %xmm6
+; AVX512F-NEXT: vptestmd %zmm3, %zmm5, %k1
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
+; AVX512F-NEXT: vpmovdb %zmm5, (%rsp)
+; AVX512F-NEXT: vptestmd %zmm3, %zmm7, %k2
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm2, %zmm4, %k2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k2} {z}
-; AVX512F-NEXT: vpmovdb %zmm4, (%rsp)
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm5
-; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm6
-; AVX512F-NEXT: vpaddd %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT: vmovd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vptestmd %zmm2, %zmm4, %k2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
-; AVX512F-NEXT: vpextrd $1, %xmm5, %eax
-; AVX512F-NEXT: vmovd %xmm5, %ecx
+; AVX512F-NEXT: vpextrd $1, %xmm6, %eax
+; AVX512F-NEXT: vmovd %xmm6, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm5
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
; AVX512F-NEXT: vpmovdb %zmm5, (%rsp,%rcx)
-; AVX512F-NEXT: vptestmd %zmm2, %zmm4, %k3
+; AVX512F-NEXT: vptestmd %zmm3, %zmm4, %k3
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm2, %zmm2 {%k3} {z}
-; AVX512F-NEXT: vpmovdb %zmm2, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k3} {z}
+; AVX512F-NEXT: vpmovdb %zmm3, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
+; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
@@ -2778,7 +2745,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
-; AVX512F-NEXT: vpsubd %zmm3, %zmm4, %zmm4
+; AVX512F-NEXT: vpsubd %zmm2, %zmm4, %zmm4
; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
@@ -2791,17 +2758,17 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: andl $63, %ecx
; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %ymm4
; AVX512F-NEXT: vmovdqa %ymm4, 64(%rsp,%rcx)
-; AVX512F-NEXT: vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
; AVX512F-NEXT: vpmovdb %zmm4, %xmm4
-; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; AVX512F-NEXT: vpblendvb %ymm2, {{[0-9]+}}(%rsp), %ymm4, %ymm2
-; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vpblendvb %ymm3, {{[0-9]+}}(%rsp), %ymm4, %ymm3
+; AVX512F-NEXT: vpmovdb %zmm2, %xmm2
; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, {{[0-9]+}}(%rsp), %ymm1, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; AVX512F-NEXT: movq %rbp, %rsp
; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
@@ -4446,42 +4413,26 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: movq %rsp, %rbp
; AVX512F-NEXT: andq $-64, %rsp
; AVX512F-NEXT: subq $576, %rsp # imm = 0x240
-; AVX512F-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 360(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, 368(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, 376(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, 384(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, 392(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 400(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 408(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 416(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 424(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 432(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 440(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 448(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 456(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 464(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $15, 472(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k1
-; AVX512F-NEXT: vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vmovdqu64 352(%rbp), %zmm4
+; AVX512F-NEXT: vmovdqu64 416(%rbp), %zmm5
+; AVX512F-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k1
+; AVX512F-NEXT: vmovdqu64 224(%rbp), %zmm4
+; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm6
+; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
+; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; AVX512F-NEXT: vmovdqu64 96(%rbp), %zmm6
+; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm7
+; AVX512F-NEXT: vpmovqb %zmm7, %xmm7
+; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm5, %zmm6, %k2
; AVX512F-NEXT: vmovd %edi, %xmm6
; AVX512F-NEXT: vpinsrb $1, %esi, %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $2, %edx, %xmm6, %xmm6
@@ -4497,44 +4448,27 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k2
-; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm6, %xmm5
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k3
+; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm6, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm5, %zmm6, %k3
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k3} {z}
; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm5
-; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm6
-; AVX512F-NEXT: vpaddd %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT: vpextrd $1, %xmm5, %eax
-; AVX512F-NEXT: vmovd %xmm5, %ecx
-; AVX512F-NEXT: vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm5, %xmm5
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm6
+; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512F-NEXT: vpaddd %ymm7, %ymm6, %ymm6
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512F-NEXT: vpaddd %xmm7, %xmm6, %xmm6
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm7, %xmm6, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpextrd $1, %xmm6, %eax
+; AVX512F-NEXT: vmovd %xmm6, %ecx
; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k2} {z}
; AVX512F-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
-; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k3
+; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k3
; AVX512F-NEXT: vpcompressd %zmm2, %zmm1 {%k3} {z}
; AVX512F-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
>From 7284023d9981a310f79eca7bd675a0d35dc45163 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 12 Jul 2026 08:05:48 +0000
Subject: [PATCH 2/3] [X86] Bound strided wide loads to the accessed range
The strided fold read up to (stride - element) bytes past the last
element. Keep the natural wide loads when the whole span is
dereferenceable or the base is a fixed stack slot, which always has the
caller frame mapped above it, or the base alignment keeps each load
inside a page. Otherwise offset the last wide load back so it ends at
the last element and srl the elements back down a lane. Without VLX the
sub-512-bit VTRUNC forms reached by the alignment and shift tiers do
not select, so those tiers stand down below 512 bits there.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 28 +++++
.../CodeGen/X86/buildvec-strided-loads.ll | 104 ++++++++++++------
2 files changed, 101 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 64d298d26b84f..731a58f6250ae 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8098,6 +8098,14 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
MVT ConcatVT = MVT::getVectorVT(SrcEltVT, NumElems);
if (NumElems > TruncDstLanes && !TLI.isTypeLegal(ConcatVT))
return SDValue();
+ // The wide loads read (stride - element) bytes past the last element.
+ // That is safe if the whole span is dereferenceable or the base is a
+ // fixed stack slot which always has the caller's frame mapped above it.
+ bool RangeSafe = isa_and_nonnull<FixedStackPseudoSourceValue>(
+ LDBase->getMemOperand()->getPseudoValue()) ||
+ LDBase->getPointerInfo().isDereferenceable(
+ NumElems * (WideEltBits / 8), *DAG.getContext(),
+ DAG.getDataLayout());
// Try wider register sizes first.
for (unsigned WideRegBits : {512u, 256u, 128u}) {
unsigned LanesPerWideLoad = WideRegBits / WideEltBits;
@@ -8115,11 +8123,27 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
(WideEltBits == 16 && !Subtarget.hasBWI())))
continue;
unsigned BytesPerWideLoad = WideRegBits / 8;
+ // VTRUNC below 512 bits needs VLX.
+ bool NarrowSafe = WideVT.is512BitVector() || !Subtarget.hasAVX512() ||
+ Subtarget.hasVLX();
+ // A load aligned to its own size stays inside one page.
+ bool NaturalSafe =
+ RangeSafe ||
+ (NarrowSafe && LDBase->getBaseAlign() >= Align(BytesPerWideLoad));
+ // Otherwise the last load is shifted back to end at the last element.
+ // The shifted load overlaps the previous one, so two loads are needed.
+ bool CanShift = NumWideLoads >= 2 && NarrowSafe;
+ if (!NaturalSafe && !CanShift)
+ continue;
+ unsigned TailBits = WideEltBits - BaseSizeInBits;
auto MMOFlags = LDBase->getMemOperand()->getFlags();
SDValue BasePtr = LDBase->getBasePtr();
SmallVector<SDValue, 8> Pieces;
for (unsigned K = 0; K != NumWideLoads; ++K) {
+ bool ShiftLast = !NaturalSafe && K == NumWideLoads - 1;
unsigned Offset = K * BytesPerWideLoad;
+ if (ShiftLast)
+ Offset -= TailBits / 8;
SDValue Ptr = K == 0 ? BasePtr
: DAG.getMemBasePlusOffset(
BasePtr, TypeSize::getFixed(Offset), DL);
@@ -8130,6 +8154,10 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
for (auto *LD : Loads)
if (LD)
DAG.makeEquivalentMemoryOrdering(LD, Ld);
+ // The shifted load leaves each element in its lane's high bits.
+ if (ShiftLast)
+ Ld = DAG.getNode(ISD::SRL, DL, WideVT, Ld,
+ DAG.getConstant(TailBits, DL, WideVT));
unsigned TruncOp = Partial ? X86ISD::VTRUNC : ISD::TRUNCATE;
Pieces.push_back(DAG.getNode(TruncOp, DL, TruncDstVT, Ld));
}
diff --git a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
index e7f848a6e9ee9..befa448ab408a 100644
--- a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
+++ b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
; AVX2-LABEL: buildvec_v16i8_stride8:
@@ -27,11 +27,11 @@ define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
;
; AVX512-LABEL: buildvec_v16i8_stride8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
-; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpsrlq $56, 57(%rdi), %zmm0
; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
@@ -107,12 +107,36 @@ define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
; AVX2-NEXT: retq
;
-; AVX512-LABEL: buildvec_v16i8_stride4:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: buildvec_v16i8_stride4:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movzbl (%rdi), %eax
+; AVX512F-NEXT: vmovd %eax, %xmm0
+; AVX512F-NEXT: vpinsrb $1, 4(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $2, 8(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $3, 12(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $4, 16(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $5, 20(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $6, 24(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $7, 28(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $8, 32(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $9, 36(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $11, 44(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $12, 48(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $13, 52(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $14, 56(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: buildvec_v16i8_stride4:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpsrld $24, 29(%rdi), %ymm0
+; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT: vmovdqu (%rdi), %ymm1
+; AVX512VL-NEXT: vpmovdb %ymm1, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
%p1 = getelementptr inbounds i8, ptr %p, i64 4
%p2 = getelementptr inbounds i8, ptr %p, i64 8
@@ -167,20 +191,38 @@ define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
define <8 x i16> @buildvec_v8i16_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v8i16_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: movzwl (%rdi), %eax
+; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrw $1, 4(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $2, 8(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $3, 12(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $4, 16(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $5, 20(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $6, 24(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrw $7, 28(%rdi), %xmm0, %xmm0
; AVX2-NEXT: retq
;
-; AVX512-LABEL: buildvec_v8i16_stride4:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqu (%rdi), %ymm0
-; AVX512-NEXT: vpmovdw %zmm0, %ymm0
-; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: buildvec_v8i16_stride4:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movzwl (%rdi), %eax
+; AVX512F-NEXT: vmovd %eax, %xmm0
+; AVX512F-NEXT: vpinsrw $1, 4(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $2, 8(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $3, 12(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $4, 16(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $5, 20(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $6, 24(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrw $7, 28(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: buildvec_v8i16_stride4:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovdqu (%rdi), %xmm0
+; AVX512VL-NEXT: vpsrld $16, 14(%rdi), %xmm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
%p1 = getelementptr inbounds i8, ptr %p, i64 4
%p2 = getelementptr inbounds i8, ptr %p, i64 8
@@ -235,8 +277,8 @@ define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
; AVX512-LABEL: buildvec_v16i16_stride8:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpsrlq $48, 58(%rdi), %zmm1
; AVX512-NEXT: vpmovqw %zmm1, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -334,8 +376,8 @@ define <32 x i8> @buildvec_v32i8_stride4(ptr %p) {
; AVX512-LABEL: buildvec_v32i8_stride4:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpsrld $24, 61(%rdi), %zmm1
; AVX512-NEXT: vpmovdb %zmm1, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT: retq
@@ -669,8 +711,8 @@ define <32 x i8> @buildvec_v32i8_stride8(ptr %p) {
; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
; AVX512-NEXT: vmovdqu64 128(%rdi), %zmm2
-; AVX512-NEXT: vmovdqu64 192(%rdi), %zmm3
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpsrlq $56, 185(%rdi), %zmm3
; AVX512-NEXT: vpmovqb %zmm3, %xmm3
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
; AVX512-NEXT: vpmovqb %zmm0, %xmm0
@@ -1015,11 +1057,11 @@ define <16 x i8> @buildvec_v16i8_stride8_reverse(ptr %p) {
;
; AVX512-LABEL: buildvec_v16i8_stride8_reverse:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
-; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpsrlq $56, 57(%rdi), %zmm0
; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
>From 2d8767ce4df80d16d8427d139a39b79d4e89c5e7 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 16 Jul 2026 04:03:17 +0000
Subject: [PATCH 3/3] [X86] Use stride alignment and emit generic truncates
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 91 ++----
llvm/test/CodeGen/X86/avx10_2bf16-fma.ll | 177 ++++------
llvm/test/CodeGen/X86/avx512-calling-conv.ll | 290 +++++++++--------
llvm/test/CodeGen/X86/avx512-ext.ll | 48 +--
.../test/CodeGen/X86/avx512-insert-extract.ll | 163 +++++-----
llvm/test/CodeGen/X86/avx512-load-store.ll | 32 +-
llvm/test/CodeGen/X86/avx512-mask-op.ll | 48 +--
.../CodeGen/X86/avx512-masked_memop-16-8.ll | 146 +++++----
llvm/test/CodeGen/X86/build-vector-128.ll | 54 ++--
llvm/test/CodeGen/X86/build-vector-256.ll | 68 ++--
llvm/test/CodeGen/X86/build-vector-512.ll | 120 ++++---
.../CodeGen/X86/buildvec-strided-loads.ll | 301 +++++++++---------
llvm/test/CodeGen/X86/vector-compress.ll | 48 +--
13 files changed, 794 insertions(+), 792 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 731a58f6250ae..1b77944bc5058 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8091,66 +8091,53 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
if (WideEltBits != 0) {
MVT WideEltVT = MVT::getIntegerVT(WideEltBits);
MVT SrcEltVT = MVT::getIntegerVT(BaseSizeInBits);
- // VTRUNC writes the truncated values to the low lanes of an xmm with
- // zero padding above.
- MVT TruncDstVT = MVT::getVectorVT(SrcEltVT, 128 / BaseSizeInBits);
- unsigned TruncDstLanes = TruncDstVT.getVectorNumElements();
- MVT ConcatVT = MVT::getVectorVT(SrcEltVT, NumElems);
- if (NumElems > TruncDstLanes && !TLI.isTypeLegal(ConcatVT))
- return SDValue();
// The wide loads read (stride - element) bytes past the last element.
- // That is safe if the whole span is dereferenceable or the base is a
- // fixed stack slot which always has the caller's frame mapped above it.
- bool RangeSafe = isa_and_nonnull<FixedStackPseudoSourceValue>(
- LDBase->getMemOperand()->getPseudoValue()) ||
- LDBase->getPointerInfo().isDereferenceable(
- NumElems * (WideEltBits / 8), *DAG.getContext(),
- DAG.getDataLayout());
- // Try wider register sizes first.
+ // A stride-aligned base keeps every stride block inside a page that an
+ // accessed element touches.
+ unsigned StrideBytes = WideEltBits / 8;
+ bool RangeSafe =
+ LDBase->getBaseAlign() >= Align(StrideBytes) ||
+ LDBase->getPointerInfo().isDereferenceable(
+ NumElems * StrideBytes, *DAG.getContext(), DAG.getDataLayout());
for (unsigned WideRegBits : {512u, 256u, 128u}) {
unsigned LanesPerWideLoad = WideRegBits / WideEltBits;
if (LanesPerWideLoad < 2 || NumElems % LanesPerWideLoad != 0)
continue;
- if (LanesPerWideLoad > TruncDstLanes)
- continue; // VTRUNC dest must hold all good lanes of one piece.
+ // Truncate to a legal piece at least 128-bit and no narrower
+ // than the source element.
+ unsigned PieceEltBits =
+ std::max(BaseSizeInBits, 128 / LanesPerWideLoad);
+ MVT PieceEltVT = MVT::getIntegerVT(PieceEltBits);
MVT WideVT = MVT::getVectorVT(WideEltVT, LanesPerWideLoad);
- if (!TLI.isTypeLegal(WideVT) || !TLI.isTypeLegal(TruncDstVT))
+ MVT PieceVT = MVT::getVectorVT(PieceEltVT, LanesPerWideLoad);
+ MVT TruncVT = MVT::getVectorVT(SrcEltVT, NumElems);
+ MVT ConcatVT = MVT::getVectorVT(PieceEltVT, NumElems);
+ if (!TLI.isTypeLegal(WideVT) || !TLI.isTypeLegal(PieceVT) ||
+ !TLI.isTypeLegal(ConcatVT) || !TLI.isTypeLegal(TruncVT))
continue;
unsigned NumWideLoads = NumElems / LanesPerWideLoad;
- // VTRUNC has no non-AVX-512 lowering so the i16 to i8 form needs BWI.
- bool Partial = LanesPerWideLoad != TruncDstLanes;
- if (Partial && (!Subtarget.hasAVX512() ||
- (WideEltBits == 16 && !Subtarget.hasBWI())))
- continue;
unsigned BytesPerWideLoad = WideRegBits / 8;
- // VTRUNC below 512 bits needs VLX.
- bool NarrowSafe = WideVT.is512BitVector() || !Subtarget.hasAVX512() ||
- Subtarget.hasVLX();
- // A load aligned to its own size stays inside one page.
- bool NaturalSafe =
- RangeSafe ||
- (NarrowSafe && LDBase->getBaseAlign() >= Align(BytesPerWideLoad));
- // Otherwise the last load is shifted back to end at the last element.
- // The shifted load overlaps the previous one, so two loads are needed.
- bool CanShift = NumWideLoads >= 2 && NarrowSafe;
- if (!NaturalSafe && !CanShift)
+ // Without a provable range the last load is shifted back to end at
+ // the last element. It overlaps the previous load so two are needed.
+ bool CanShift = NumWideLoads >= 2;
+ if (!RangeSafe && !CanShift)
continue;
unsigned TailBits = WideEltBits - BaseSizeInBits;
auto MMOFlags = LDBase->getMemOperand()->getFlags();
SDValue BasePtr = LDBase->getBasePtr();
SmallVector<SDValue, 8> Pieces;
for (unsigned K = 0; K != NumWideLoads; ++K) {
- bool ShiftLast = !NaturalSafe && K == NumWideLoads - 1;
+ bool ShiftLast = !RangeSafe && K == NumWideLoads - 1;
unsigned Offset = K * BytesPerWideLoad;
if (ShiftLast)
Offset -= TailBits / 8;
- SDValue Ptr = K == 0 ? BasePtr
- : DAG.getMemBasePlusOffset(
- BasePtr, TypeSize::getFixed(Offset), DL);
+ SDValue Ptr =
+ DAG.getMemBasePlusOffset(BasePtr, TypeSize::getFixed(Offset), DL);
+ Align LdAlign = commonAlignment(LDBase->getBaseAlign(), Offset);
SDValue Ld =
DAG.getLoad(WideVT, DL, LDBase->getChain(), Ptr,
LDBase->getPointerInfo().getWithOffset(Offset),
- K == 0 ? LDBase->getBaseAlign() : Align(1), MMOFlags);
+ LdAlign, MMOFlags);
for (auto *LD : Loads)
if (LD)
DAG.makeEquivalentMemoryOrdering(LD, Ld);
@@ -8158,32 +8145,10 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
if (ShiftLast)
Ld = DAG.getNode(ISD::SRL, DL, WideVT, Ld,
DAG.getConstant(TailBits, DL, WideVT));
- unsigned TruncOp = Partial ? X86ISD::VTRUNC : ISD::TRUNCATE;
- Pieces.push_back(DAG.getNode(TruncOp, DL, TruncDstVT, Ld));
- }
- // Pairwise shuffle the low halves until each piece is full.
- if (Partial) {
- unsigned GoodLanes = LanesPerWideLoad;
- while (Pieces.size() > 1 && GoodLanes < TruncDstLanes) {
- assert((TruncDstLanes % GoodLanes) == 0 &&
- "Illegal VTRUNC packing");
- SmallVector<SDValue, 8> Next;
- SmallVector<int, 16> Mask(TruncDstLanes, -1);
- for (unsigned I = 0; I != GoodLanes; ++I) {
- Mask[I] = I;
- Mask[GoodLanes + I] = TruncDstLanes + I;
- }
- for (unsigned J = 0, E = Pieces.size() - 1; J < E; J += 2)
- Next.push_back(DAG.getVectorShuffle(TruncDstVT, DL, Pieces[J],
- Pieces[J + 1], Mask));
- Pieces = std::move(Next);
- GoodLanes *= 2;
- }
+ Pieces.push_back(DAG.getNode(ISD::TRUNCATE, DL, PieceVT, Ld));
}
- if (Pieces.size() == 1)
- return DAG.getBitcast(VT, Pieces[0]);
-
SDValue Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Pieces);
+ Result = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Result);
return DAG.getBitcast(VT, Result);
}
}
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
index d79f0cc79b5f4..26cf3aba60407 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
@@ -647,155 +647,100 @@ define <29 x bfloat> @fuse_v19bf16_load(<29 x bfloat> %x, <29 x bfloat> %y, ptr
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm9
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1],xmm11[2],xmm13[2],xmm11[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1],xmm10[2],xmm13[2],xmm10[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],xmm11[0]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm9 = xmm9[0],xmm11[0]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm10, %ymm10
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm9, %ymm9
-; AVXNECONVERT-NEXT: vmulps %ymm10, %ymm9, %ymm9
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm9 = xmm9[0],xmm10[0],zero,zero
+; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm9 = xmm9[0,1],xmm10[0],xmm9[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm10 = xmm11[0],xmm10[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm11
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm12
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm13
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1]
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],xmm11[0]
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm10 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm10, %ymm10
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3]
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm8 = xmm8[0],xmm10[0],zero,zero
+; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm10
+; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm8 = xmm8[0,1],xmm10[0],xmm8[3]
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm9, %ymm9
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm8, %ymm8
-; AVXNECONVERT-NEXT: vmulps %ymm10, %ymm8, %ymm8
+; AVXNECONVERT-NEXT: vmulps %ymm9, %ymm8, %ymm8
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm4
-; AVXNECONVERT-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm1
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],zero,zero
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm2
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm3[0],xmm2[0],zero,zero
-; AVXNECONVERT-NEXT: vpinsrw $0, {{[0-9]+}}(%rsp), %xmm0, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm8, %xmm3
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm1
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vmulps %ymm0, %ymm2, %ymm0
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm2
+; AVXNECONVERT-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm3
+; AVXNECONVERT-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm4
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm3 = xmm3[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vshufps {{.*#+}} xmm4 = xmm4[0,2],mem[0,2]
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
+; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: vmulps %ymm2, %ymm3, %ymm2
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
+; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm2, %ymm1
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vaddps %ymm2, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm9, %xmm3
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm8, %xmm3
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm2
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm2, %ymm2
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm2, %ymm2
-; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVXNECONVERT-NEXT: vpslld $16, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVXNECONVERT-NEXT: vpslld $16, %ymm0, %ymm0
; AVXNECONVERT-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVXNECONVERT-NEXT: vpslld $16, %ymm3, %ymm3
-; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm1, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm1, 56(%rdi)
-; AVXNECONVERT-NEXT: vmovq %xmm1, 48(%rdi)
-; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm1
-; AVXNECONVERT-NEXT: vmovaps %xmm1, 32(%rdi)
-; AVXNECONVERT-NEXT: vmovaps %ymm0, (%rdi)
+; AVXNECONVERT-NEXT: vaddps %ymm3, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNECONVERT-NEXT: vmovaps %xmm0, 32(%rdi)
+; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm2, %xmm0
+; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, 56(%rdi)
+; AVXNECONVERT-NEXT: vmovq %xmm0, 48(%rdi)
+; AVXNECONVERT-NEXT: vmovaps %ymm1, (%rdi)
; AVXNECONVERT-NEXT: vzeroupper
; AVXNECONVERT-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index 2e7425c13009f..0a19a36f5cb03 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -682,10 +682,10 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; KNL-NEXT: movq %rdi, %rax
; KNL-NEXT: vmovdqu64 152(%rsp), %zmm0
; KNL-NEXT: vmovdqu64 216(%rsp), %zmm1
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; KNL-NEXT: vmovd %esi, %xmm1
; KNL-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1
; KNL-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1
@@ -702,7 +702,7 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; KNL-NEXT: vpinsrb $13, 120(%rsp), %xmm1, %xmm1
; KNL-NEXT: vpinsrb $14, 128(%rsp), %xmm1, %xmm1
; KNL-NEXT: vpinsrb $15, 136(%rsp), %xmm1, %xmm1
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; KNL-NEXT: kmovw 280(%rsp), %k0
; KNL-NEXT: kmovw 144(%rsp), %k1
; KNL-NEXT: kandw %k0, %k1, %k1
@@ -803,14 +803,15 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; SKX-NEXT: pushq %r13
; SKX-NEXT: pushq %r12
; SKX-NEXT: pushq %rbx
-; SKX-NEXT: movq %rdi, %rax
; SKX-NEXT: vmovdqu64 152(%rsp), %zmm0
; SKX-NEXT: vmovdqu64 216(%rsp), %zmm1
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vpmovqb %zmm0, %xmm0
-; SKX-NEXT: vpbroadcastb 280(%rsp), %xmm2
-; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SKX-NEXT: vpmovqw %zmm0, %xmm0
+; SKX-NEXT: vpmovqw %zmm1, %xmm1
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT: vpmovwb %ymm0, %xmm0
+; SKX-NEXT: vpbroadcastb 280(%rsp), %xmm1
+; SKX-NEXT: movq %rdi, %rax
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; SKX-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; SKX-NEXT: vptestmb %ymm1, %ymm0, %k1
; SKX-NEXT: vmovd %esi, %xmm0
@@ -1027,10 +1028,11 @@ define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
; FASTISEL-NEXT: pushq %rbx
; FASTISEL-NEXT: vmovdqu64 152(%rsp), %zmm0
; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm1
-; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
+; FASTISEL-NEXT: vpmovqw %zmm0, %xmm0
+; FASTISEL-NEXT: vpmovqw %zmm1, %xmm1
; FASTISEL-NEXT: movq %rdi, %rax
-; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
-; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; FASTISEL-NEXT: vpmovwb %ymm0, %xmm0
; FASTISEL-NEXT: vpbroadcastb 280(%rsp), %xmm1
; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; FASTISEL-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
@@ -2301,57 +2303,57 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
; KNL-NEXT: pushq %rbx
; KNL-NEXT: vmovdqu64 480(%rsp), %zmm0
; KNL-NEXT: vmovdqu64 544(%rsp), %zmm1
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: kmovw %k0, %r10d
; KNL-NEXT: vmovdqu64 608(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 672(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %r11d
-; KNL-NEXT: shll $16, %r11d
-; KNL-NEXT: orl %eax, %r11d
+; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: shll $16, %eax
+; KNL-NEXT: orl %r10d, %eax
; KNL-NEXT: vmovdqu64 736(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 800(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
-; KNL-NEXT: kmovw %k0, %eax
+; KNL-NEXT: kmovw %k0, %r11d
; KNL-NEXT: vmovdqu64 864(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 928(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
; KNL-NEXT: kmovw %k0, %r10d
; KNL-NEXT: shll $16, %r10d
-; KNL-NEXT: orl %eax, %r10d
+; KNL-NEXT: orl %r11d, %r10d
; KNL-NEXT: shlq $32, %r10
-; KNL-NEXT: orq %r11, %r10
+; KNL-NEXT: orq %rax, %r10
; KNL-NEXT: vmovdqu64 224(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 288(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
; KNL-NEXT: kmovw %k0, %eax
; KNL-NEXT: vmovdqu64 352(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 416(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
; KNL-NEXT: kmovw %k0, %r11d
; KNL-NEXT: shll $16, %r11d
@@ -2359,10 +2361,10 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
; KNL-NEXT: shlq $32, %r11
; KNL-NEXT: vmovdqu64 96(%rsp), %zmm1
; KNL-NEXT: vmovdqu64 160(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
; KNL-NEXT: vptestmd %zmm0, %zmm1, %k0
; KNL-NEXT: kmovw %k0, %ebx
; KNL-NEXT: vmovd %edi, %xmm1
@@ -2393,65 +2395,69 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
;
; SKX-LABEL: PR179334:
; SKX: ## %bb.0:
-; SKX-NEXT: vmovdqu64 216(%rsp), %zmm0
-; SKX-NEXT: vmovdqu64 280(%rsp), %zmm1
-; SKX-NEXT: vmovdqu64 344(%rsp), %zmm2
-; SKX-NEXT: vmovdqu64 408(%rsp), %zmm3
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vpmovqb %zmm0, %xmm0
-; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; SKX-NEXT: vpmovqb %zmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SKX-NEXT: vmovdqu64 88(%rsp), %zmm2
-; SKX-NEXT: vmovdqu64 152(%rsp), %zmm3
-; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vpmovqb %zmm2, %xmm2
-; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; SKX-NEXT: vmovd %edi, %xmm1
-; SKX-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $6, 8(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $7, 16(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $8, 24(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $9, 32(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $10, 40(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $11, 48(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $12, 56(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $13, 64(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpinsrb $14, 72(%rsp), %xmm1, %xmm1
-; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SKX-NEXT: vpinsrb $15, 80(%rsp), %xmm1, %xmm1
+; SKX-NEXT: vmovd %edi, %xmm0
+; SKX-NEXT: vpinsrb $1, %esi, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $6, 8(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $7, 16(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $8, 24(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $9, 32(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $10, 40(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $11, 48(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $12, 56(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $13, 64(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $14, 72(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $15, 80(%rsp), %xmm0, %xmm0
+; SKX-NEXT: vmovdqu64 88(%rsp), %zmm1
+; SKX-NEXT: vmovdqu64 152(%rsp), %zmm2
+; SKX-NEXT: vpmovqw %zmm1, %xmm1
+; SKX-NEXT: vpmovqw %zmm2, %xmm2
; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vpmovwb %ymm1, %xmm1
+; SKX-NEXT: vmovdqu64 216(%rsp), %zmm2
+; SKX-NEXT: vmovdqu64 280(%rsp), %zmm3
+; SKX-NEXT: vmovdqu64 344(%rsp), %zmm4
+; SKX-NEXT: vmovdqu64 408(%rsp), %zmm5
+; SKX-NEXT: vpmovqw %zmm4, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT: vpmovqw %zmm5, %xmm1
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm4, %ymm1
+; SKX-NEXT: vpmovqw %zmm2, %xmm2
+; SKX-NEXT: vpmovqw %zmm3, %xmm3
+; SKX-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; SKX-NEXT: vpmovwb %zmm1, %ymm1
; SKX-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX-NEXT: vptestmb %zmm2, %zmm0, %k0
-; SKX-NEXT: vmovdqu64 792(%rsp), %zmm0
-; SKX-NEXT: vpmovqb %zmm0, %xmm0
-; SKX-NEXT: vmovdqu64 920(%rsp), %zmm1
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
; SKX-NEXT: vmovdqu64 472(%rsp), %zmm3
; SKX-NEXT: vmovdqu64 536(%rsp), %zmm4
; SKX-NEXT: vmovdqu64 600(%rsp), %zmm5
; SKX-NEXT: vmovdqu64 664(%rsp), %zmm6
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; SKX-NEXT: vpmovqb %zmm4, %xmm1
-; SKX-NEXT: vpmovqb %zmm6, %xmm4
-; SKX-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
-; SKX-NEXT: vmovdqu64 728(%rsp), %zmm4
-; SKX-NEXT: vpmovqb %zmm4, %xmm4
-; SKX-NEXT: vmovdqu64 856(%rsp), %zmm6
-; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; SKX-NEXT: vpmovqb %zmm6, %xmm1
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm4, %ymm1
-; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vpmovqb %zmm5, %xmm4
+; SKX-NEXT: vpmovqw %zmm5, %xmm5
+; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; SKX-NEXT: vpmovqw %zmm6, %xmm1
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm5, %ymm1
+; SKX-NEXT: vpmovqw %zmm3, %xmm3
+; SKX-NEXT: vpmovqw %zmm4, %xmm4
; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
-; SKX-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; SKX-NEXT: vpmovwb %zmm1, %ymm1
+; SKX-NEXT: vmovdqu64 728(%rsp), %zmm3
+; SKX-NEXT: vmovdqu64 792(%rsp), %zmm4
+; SKX-NEXT: vmovdqu64 856(%rsp), %zmm5
+; SKX-NEXT: vmovdqu64 920(%rsp), %zmm6
+; SKX-NEXT: vpmovqw %zmm5, %xmm5
+; SKX-NEXT: vpmovqw %zmm6, %xmm6
+; SKX-NEXT: vptestmb %zmm2, %zmm0, %k0
+; SKX-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm0
+; SKX-NEXT: vpmovqw %zmm3, %xmm3
+; SKX-NEXT: vpmovqw %zmm4, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; SKX-NEXT: vpmovwb %zmm0, %ymm0
+; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; SKX-NEXT: vptestmb %zmm2, %zmm0, %k1
; SKX-NEXT: kmovq %k1, %rdx
; SKX-NEXT: kmovq %k0, %rax
@@ -2482,46 +2488,32 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
;
; FASTISEL-LABEL: PR179334:
; FASTISEL: ## %bb.0:
-; FASTISEL-NEXT: vmovdqu64 792(%rsp), %zmm0
-; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
-; FASTISEL-NEXT: vmovdqu64 920(%rsp), %zmm1
-; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
-; FASTISEL-NEXT: vmovdqu64 472(%rsp), %zmm2
-; FASTISEL-NEXT: vmovdqu64 536(%rsp), %zmm3
-; FASTISEL-NEXT: vmovdqu64 600(%rsp), %zmm4
-; FASTISEL-NEXT: vmovdqu64 664(%rsp), %zmm5
-; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vpmovqb %zmm5, %xmm5
-; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; FASTISEL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm1
-; FASTISEL-NEXT: vmovdqu64 728(%rsp), %zmm3
-; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; FASTISEL-NEXT: vmovdqu64 856(%rsp), %zmm1
-; FASTISEL-NEXT: vpmovqb %zmm1, %xmm1
-; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; FASTISEL-NEXT: vpmovqb %zmm2, %xmm2
-; FASTISEL-NEXT: vpmovqb %zmm4, %xmm3
+; FASTISEL-NEXT: vmovdqu64 472(%rsp), %zmm0
+; FASTISEL-NEXT: vmovdqu64 536(%rsp), %zmm1
+; FASTISEL-NEXT: vmovdqu64 600(%rsp), %zmm2
+; FASTISEL-NEXT: vmovdqu64 664(%rsp), %zmm3
+; FASTISEL-NEXT: vpmovqw %zmm2, %xmm2
+; FASTISEL-NEXT: vpmovqw %zmm3, %xmm3
; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; FASTISEL-NEXT: vpmovqw %zmm0, %xmm0
+; FASTISEL-NEXT: vpmovqw %zmm1, %xmm1
+; FASTISEL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; FASTISEL-NEXT: vpmovwb %zmm0, %ymm0
+; FASTISEL-NEXT: vmovdqu64 728(%rsp), %zmm1
+; FASTISEL-NEXT: vmovdqu64 792(%rsp), %zmm2
+; FASTISEL-NEXT: vmovdqu64 856(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 920(%rsp), %zmm4
+; FASTISEL-NEXT: vpmovqw %zmm3, %xmm3
+; FASTISEL-NEXT: vpmovqw %zmm4, %xmm4
+; FASTISEL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; FASTISEL-NEXT: vpmovqw %zmm1, %xmm1
+; FASTISEL-NEXT: vpmovqw %zmm2, %xmm2
+; FASTISEL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; FASTISEL-NEXT: vpmovwb %zmm1, %ymm1
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; FASTISEL-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k0
-; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm0
-; FASTISEL-NEXT: vmovdqu64 280(%rsp), %zmm2
-; FASTISEL-NEXT: vmovdqu64 344(%rsp), %zmm3
-; FASTISEL-NEXT: vmovdqu64 408(%rsp), %zmm4
-; FASTISEL-NEXT: vpmovqb %zmm2, %xmm2
-; FASTISEL-NEXT: vpmovqb %zmm4, %xmm4
-; FASTISEL-NEXT: vpmovqb %zmm0, %xmm0
-; FASTISEL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
-; FASTISEL-NEXT: vmovdqu64 88(%rsp), %zmm3
-; FASTISEL-NEXT: vmovdqu64 152(%rsp), %zmm4
-; FASTISEL-NEXT: vpmovqb %zmm4, %xmm4
-; FASTISEL-NEXT: vpmovqb %zmm3, %xmm3
-; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; FASTISEL-NEXT: vmovd %edi, %xmm2
; FASTISEL-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
; FASTISEL-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
@@ -2537,10 +2529,28 @@ define i128 @PR179334(<128 x i1> %m) nounwind {
; FASTISEL-NEXT: vpinsrb $12, 56(%rsp), %xmm2, %xmm2
; FASTISEL-NEXT: vpinsrb $13, 64(%rsp), %xmm2, %xmm2
; FASTISEL-NEXT: vpinsrb $14, 72(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
; FASTISEL-NEXT: vpinsrb $15, 80(%rsp), %xmm2, %xmm2
-; FASTISEL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; FASTISEL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; FASTISEL-NEXT: vmovdqu64 88(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 152(%rsp), %zmm4
+; FASTISEL-NEXT: vpmovqw %zmm3, %xmm3
+; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k0
+; FASTISEL-NEXT: vpmovqw %zmm4, %xmm0
+; FASTISEL-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm0
+; FASTISEL-NEXT: vpmovwb %ymm0, %xmm0
+; FASTISEL-NEXT: vmovdqu64 216(%rsp), %zmm3
+; FASTISEL-NEXT: vmovdqu64 280(%rsp), %zmm4
+; FASTISEL-NEXT: vmovdqu64 344(%rsp), %zmm5
+; FASTISEL-NEXT: vmovdqu64 408(%rsp), %zmm6
+; FASTISEL-NEXT: vpmovqw %zmm5, %xmm5
+; FASTISEL-NEXT: vpmovqw %zmm6, %xmm6
+; FASTISEL-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
+; FASTISEL-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm2
+; FASTISEL-NEXT: vpmovqw %zmm3, %xmm3
+; FASTISEL-NEXT: vpmovqw %zmm4, %xmm4
+; FASTISEL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; FASTISEL-NEXT: vpmovwb %zmm2, %ymm2
+; FASTISEL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; FASTISEL-NEXT: vptestmb %zmm1, %zmm0, %k1
; FASTISEL-NEXT: kmovq %k1, %rax
; FASTISEL-NEXT: kmovq %k0, %rdx
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index b4ccc1b0f610d..8315038c2e78f 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -1865,10 +1865,10 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; KNL: # %bb.0:
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; KNL-NEXT: vpmovqb %zmm3, %xmm3
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vpmovqw %zmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; KNL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k1
; KNL-NEXT: vmovd %edi, %xmm2
@@ -1891,17 +1891,17 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k2
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; KNL-NEXT: vpmovqb %zmm4, %xmm4
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vpmovqw %zmm4, %xmm4
+; KNL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k3
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; KNL-NEXT: vpmovqb %zmm4, %xmm4
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vpmovqw %zmm4, %xmm4
+; KNL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; KNL-NEXT: vptestmd %zmm3, %zmm2, %k4
; KNL-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
; KNL-NEXT: vpmovdw %zmm2, %ymm2
@@ -1930,10 +1930,10 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; AVX512DQNOBW: # %bb.0:
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; AVX512DQNOBW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512DQNOBW-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512DQNOBW-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512DQNOBW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; AVX512DQNOBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k0
; AVX512DQNOBW-NEXT: vmovd %edi, %xmm2
@@ -1956,17 +1956,17 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k1
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; AVX512DQNOBW-NEXT: vpmovqb %zmm4, %xmm4
-; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512DQNOBW-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQNOBW-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; AVX512DQNOBW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k2
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; AVX512DQNOBW-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; AVX512DQNOBW-NEXT: vpmovqb %zmm4, %xmm4
-; AVX512DQNOBW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQNOBW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; AVX512DQNOBW-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512DQNOBW-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQNOBW-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQNOBW-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; AVX512DQNOBW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; AVX512DQNOBW-NEXT: vptestmd %zmm3, %zmm2, %k3
; AVX512DQNOBW-NEXT: vpmovm2d %k3, %zmm2
; AVX512DQNOBW-NEXT: vpmovdw %zmm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index c905082b42233..af1d3c3415d88 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -1736,37 +1736,43 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
; KNL-NEXT: vmovdqu64 288(%rbp), %zmm1
; KNL-NEXT: vmovdqu64 352(%rbp), %zmm2
; KNL-NEXT: vmovdqu64 416(%rbp), %zmm3
-; KNL-NEXT: vpmovqb %zmm1, %xmm1
-; KNL-NEXT: vpmovqb %zmm3, %xmm3
-; KNL-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; KNL-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm1, %xmm1
+; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; KNL-NEXT: vpmovdb %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm2, %xmm1
+; KNL-NEXT: vpmovqw %zmm3, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; KNL-NEXT: vpmovdb %zmm1, %xmm1
+; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
; KNL-NEXT: vpcmpeqb %ymm0, %ymm1, %ymm1
-; KNL-NEXT: vmovdqu64 96(%rbp), %zmm2
-; KNL-NEXT: vmovdqu64 160(%rbp), %zmm3
-; KNL-NEXT: vpmovqb %zmm3, %xmm3
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; KNL-NEXT: vmovd %edi, %xmm3
-; KNL-NEXT: vpinsrb $1, %esi, %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $3, %ecx, %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $4, %r8d, %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $5, %r9d, %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $6, 16(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $7, 24(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $8, 32(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $9, 40(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $10, 48(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $11, 56(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $12, 64(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $13, 72(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $14, 80(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vpinsrb $15, 88(%rbp), %xmm3, %xmm3
-; KNL-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
+; KNL-NEXT: vmovd %edi, %xmm2
+; KNL-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $6, 16(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $7, 24(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $8, 32(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $9, 40(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $10, 48(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $11, 56(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $12, 64(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $13, 72(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $14, 80(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vpinsrb $15, 88(%rbp), %xmm2, %xmm2
+; KNL-NEXT: vmovdqu64 96(%rbp), %zmm3
+; KNL-NEXT: vmovdqu64 160(%rbp), %zmm4
+; KNL-NEXT: vpmovqw %zmm3, %xmm3
+; KNL-NEXT: vpmovqw %zmm4, %xmm4
+; KNL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; KNL-NEXT: vpmovdb %zmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2
; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; KNL-NEXT: vpternlogq {{.*#+}} zmm1 = ~zmm1
@@ -1774,13 +1780,17 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
; KNL-NEXT: vmovdqu64 544(%rbp), %zmm3
; KNL-NEXT: vmovdqu64 608(%rbp), %zmm4
; KNL-NEXT: vmovdqu64 672(%rbp), %zmm5
-; KNL-NEXT: vpmovqb %zmm3, %xmm3
-; KNL-NEXT: vpmovqb %zmm5, %xmm5
-; KNL-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm4, %xmm4
-; KNL-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; KNL-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2]
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vpmovqw %zmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; KNL-NEXT: vpmovdb %zmm2, %xmm2
+; KNL-NEXT: vpmovqw %zmm4, %xmm3
+; KNL-NEXT: vpmovqw %zmm5, %xmm4
+; KNL-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; KNL-NEXT: vpmovdb %zmm3, %xmm3
+; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2
; KNL-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2
; KNL-NEXT: cmpb $0, 736(%rbp)
@@ -1843,51 +1853,54 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n
; SKX-NEXT: movq %rsp, %rbp
; SKX-NEXT: andq $-64, %rsp
; SKX-NEXT: subq $192, %rsp
-; SKX-NEXT: vmovdqu64 224(%rbp), %zmm0
-; SKX-NEXT: vmovdqu64 288(%rbp), %zmm1
-; SKX-NEXT: vmovdqu64 352(%rbp), %zmm2
-; SKX-NEXT: vmovdqu64 416(%rbp), %zmm3
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; SKX-NEXT: vpmovqb %zmm0, %xmm0
-; SKX-NEXT: vpmovqb %zmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; SKX-NEXT: vmovd %edi, %xmm0
+; SKX-NEXT: vpinsrb $1, %esi, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $6, 16(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $7, 24(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $8, 32(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $9, 40(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $10, 48(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $11, 56(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $12, 64(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $13, 72(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $14, 80(%rbp), %xmm0, %xmm0
+; SKX-NEXT: vpinsrb $15, 88(%rbp), %xmm0, %xmm0
; SKX-NEXT: vmovdqu64 96(%rbp), %zmm1
; SKX-NEXT: vmovdqu64 160(%rbp), %zmm2
-; SKX-NEXT: vpmovqb %zmm2, %xmm2
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SKX-NEXT: vmovd %edi, %xmm2
-; SKX-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $6, 16(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $7, 24(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $8, 32(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $9, 40(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $10, 48(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $11, 56(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $12, 64(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $13, 72(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $14, 80(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vpinsrb $15, 88(%rbp), %xmm2, %xmm2
-; SKX-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; SKX-NEXT: vpmovqw %zmm1, %xmm1
+; SKX-NEXT: vpmovqw %zmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vpmovwb %ymm1, %xmm1
+; SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; SKX-NEXT: vmovdqu64 224(%rbp), %zmm1
+; SKX-NEXT: vmovdqu64 288(%rbp), %zmm2
+; SKX-NEXT: vmovdqu64 352(%rbp), %zmm3
+; SKX-NEXT: vmovdqu64 416(%rbp), %zmm4
+; SKX-NEXT: vpmovqw %zmm3, %xmm3
+; SKX-NEXT: vpmovqw %zmm4, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; SKX-NEXT: vpmovqw %zmm1, %xmm1
+; SKX-NEXT: vpmovqw %zmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; SKX-NEXT: vpmovwb %zmm1, %ymm1
+; SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; SKX-NEXT: vmovdqu64 480(%rbp), %zmm1
; SKX-NEXT: vmovdqu64 544(%rbp), %zmm2
; SKX-NEXT: vmovdqu64 608(%rbp), %zmm3
; SKX-NEXT: vmovdqu64 672(%rbp), %zmm4
-; SKX-NEXT: vpmovqb %zmm2, %xmm2
-; SKX-NEXT: vpmovqb %zmm4, %xmm4
-; SKX-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; SKX-NEXT: vpmovqb %zmm1, %xmm1
-; SKX-NEXT: vpmovqb %zmm3, %xmm3
-; SKX-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; SKX-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
+; SKX-NEXT: vpmovqw %zmm3, %xmm3
+; SKX-NEXT: vpmovqw %zmm4, %xmm4
+; SKX-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; SKX-NEXT: vpmovqw %zmm1, %xmm1
+; SKX-NEXT: vpmovqw %zmm2, %xmm2
+; SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; SKX-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; SKX-NEXT: vpmovwb %zmm1, %ymm1
; SKX-NEXT: movl 744(%rbp), %eax
; SKX-NEXT: andl $127, %eax
; SKX-NEXT: vptestmb %zmm0, %zmm0, %k0
diff --git a/llvm/test/CodeGen/X86/avx512-load-store.ll b/llvm/test/CodeGen/X86/avx512-load-store.ll
index fef164c87b44b..f9adbc1b773a4 100644
--- a/llvm/test/CodeGen/X86/avx512-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx512-load-store.ll
@@ -376,32 +376,32 @@ define <80 x i32> @test_maskz_load_v80i32(ptr %p, <80 x i1> %mask) nounwind {
; CHECK64-NEXT: movq %rdi, %rax
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
-; CHECK64-NEXT: vpmovqb %zmm1, %xmm1
-; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vpmovqw %zmm0, %xmm0
+; CHECK64-NEXT: vpmovqw %zmm1, %xmm1
+; CHECK64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; CHECK64-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; CHECK64-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k1
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vpmovqw %zmm0, %xmm0
+; CHECK64-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK64-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK64-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k2
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vpmovqw %zmm0, %xmm0
+; CHECK64-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK64-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK64-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k3
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; CHECK64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK64-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK64-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK64-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK64-NEXT: vpmovqw %zmm0, %xmm0
+; CHECK64-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK64-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK64-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; CHECK64-NEXT: vptestmd %zmm1, %zmm0, %k4
; CHECK64-NEXT: vmovd %edx, %xmm0
; CHECK64-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index 57eaadf916960..d95c6d93ec8ab 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -2916,24 +2916,24 @@ define void @store_64i1(ptr %a, <64 x i1> %v) {
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k0
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k1
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k2
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; KNL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; KNL-NEXT: vpmovqb %zmm2, %xmm2
-; KNL-NEXT: vpmovqb %zmm0, %xmm0
-; KNL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; KNL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; KNL-NEXT: vpmovqw %zmm0, %xmm0
+; KNL-NEXT: vpmovqw %zmm2, %xmm2
+; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; KNL-NEXT: vptestmd %zmm1, %zmm0, %k3
; KNL-NEXT: kmovw %k3, 6(%rdi)
; KNL-NEXT: kmovw %k2, 4(%rdi)
@@ -2981,24 +2981,24 @@ define void @store_64i1(ptr %a, <64 x i1> %v) {
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k0
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k1
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k2
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
; AVX512DQ-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512DQ-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512DQ-NEXT: vptestmd %zmm1, %zmm0, %k3
; AVX512DQ-NEXT: kmovw %k3, 6(%rdi)
; AVX512DQ-NEXT: kmovw %k2, 4(%rdi)
diff --git a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
index b74771c825780..787057f01c2a7 100644
--- a/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked_memop-16-8.ll
@@ -183,90 +183,96 @@ declare void @llvm.masked.store.v16f16.p0(<16 x half>, ptr, i32, <16 x i1>)
define void @test_maskz_store_v192i8(ptr %p0, <192 x i1> %mask) nounwind {
; CHECK-LABEL: test_maskz_store_v192i8:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; CHECK-NEXT: vmovd %esi, %xmm0
+; CHECK-NEXT: vpinsrb $1, %edx, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $3, %r8d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $4, %r9d, %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; CHECK-NEXT: vpmovqb %zmm1, %xmm1
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; CHECK-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; CHECK-NEXT: vpmovqw %zmm1, %xmm1
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vpmovwb %ymm1, %xmm1
+; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vpmovqb %zmm1, %xmm1
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; CHECK-NEXT: vmovd %esi, %xmm2
-; CHECK-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; CHECK-NEXT: vpbroadcastb {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k1
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; CHECK-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; CHECK-NEXT: vpmovqw %zmm3, %xmm3
+; CHECK-NEXT: vpmovqw %zmm4, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqw %zmm1, %xmm1
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; CHECK-NEXT: vpmovwb %zmm1, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1
+; CHECK-NEXT: vpbroadcastb {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vpmovqw %zmm3, %xmm3
+; CHECK-NEXT: vpmovqw %zmm4, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqw %zmm1, %xmm1
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; CHECK-NEXT: vpmovwb %zmm1, %ymm1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vpmovqb %zmm4, %xmm4
-; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
-; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k2
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; CHECK-NEXT: vpmovqb %zmm0, %xmm0
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: vpmovqw %zmm4, %xmm4
+; CHECK-NEXT: vpmovqw %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vpmovqw %zmm3, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; CHECK-NEXT: vpmovwb %zmm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k2
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
-; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; CHECK-NEXT: vpmovqw %zmm3, %xmm3
+; CHECK-NEXT: vpmovqw %zmm4, %xmm4
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; CHECK-NEXT: vpmovqw %zmm1, %xmm1
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; CHECK-NEXT: vpmovwb %zmm1, %ymm1
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; CHECK-NEXT: vpmovqb %zmm3, %xmm3
+; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
; CHECK-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm5
-; CHECK-NEXT: vpmovqb %zmm5, %xmm5
-; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3
-; CHECK-NEXT: vpmovqb %zmm2, %xmm2
-; CHECK-NEXT: vpmovqb %zmm4, %xmm4
-; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
-; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
-; CHECK-NEXT: vptestmb %zmm1, %zmm0, %k3
+; CHECK-NEXT: vpmovqw %zmm4, %xmm4
+; CHECK-NEXT: vpmovqw %zmm5, %xmm5
+; CHECK-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
+; CHECK-NEXT: vpmovqw %zmm2, %xmm2
+; CHECK-NEXT: vpmovqw %zmm3, %xmm3
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; CHECK-NEXT: vpmovwb %zmm2, %ymm2
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; CHECK-NEXT: vptestmb %zmm0, %zmm1, %k3
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vmovdqu8 %zmm0, 64(%rdi) {%k3}
; CHECK-NEXT: vmovdqu8 %zmm0, 128(%rdi) {%k2}
diff --git a/llvm/test/CodeGen/X86/build-vector-128.ll b/llvm/test/CodeGen/X86/build-vector-128.ll
index e478e7037463a..f595178e1ed20 100644
--- a/llvm/test/CodeGen/X86/build-vector-128.ll
+++ b/llvm/test/CodeGen/X86/build-vector-128.ll
@@ -361,25 +361,25 @@ define <16 x i8> @test_buildvector_v16i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; SSE41-64-NEXT: pinsrb $15, {{[0-9]+}}(%rsp), %xmm0
; SSE41-64-NEXT: retq
;
-; AVX-32-LABEL: test_buildvector_v16i8:
-; AVX-32: # %bb.0:
-; AVX-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT: retl
+; AVX1-32-LABEL: test_buildvector_v16i8:
+; AVX1-32: # %bb.0:
+; AVX1-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX1-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX1-32-NEXT: retl
;
; AVX-64-LABEL: test_buildvector_v16i8:
; AVX-64: # %bb.0:
@@ -400,6 +400,22 @@ define <16 x i8> @test_buildvector_v16i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX-64-NEXT: retq
+;
+; AVX2-32-LABEL: test_buildvector_v16i8:
+; AVX2-32: # %bb.0:
+; AVX2-32-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm2
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm2, %ymm2
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-32-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; AVX2-32-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-32-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-32-NEXT: vzeroupper
+; AVX2-32-NEXT: retl
%ins0 = insertelement <16 x i8> undef, i8 %a0, i32 0
%ins1 = insertelement <16 x i8> %ins0, i8 %a1, i32 1
%ins2 = insertelement <16 x i8> %ins1, i8 %a2, i32 2
diff --git a/llvm/test/CodeGen/X86/build-vector-256.ll b/llvm/test/CodeGen/X86/build-vector-256.ll
index 2de346aee5857..b60e9f3dd3ee3 100644
--- a/llvm/test/CodeGen/X86/build-vector-256.ll
+++ b/llvm/test/CodeGen/X86/build-vector-256.ll
@@ -191,14 +191,13 @@ define <16 x i16> @test_buildvector_v16i16(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i
; AVX2-64-NEXT: vpinsrw $5, %r9d, %xmm0, %xmm0
; AVX2-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX2-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-64-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-64-NEXT: vpinsrw $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $4, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $5, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
-; AVX2-64-NEXT: vpinsrw $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX2-64-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm1
+; AVX2-64-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm2
+; AVX2-64-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],mem[0,2]
+; AVX2-64-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVX2-64-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-64-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-64-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX2-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-64-NEXT: retq
%ins0 = insertelement <16 x i16> undef, i16 %a0, i32 0
@@ -297,39 +296,26 @@ define <32 x i8> @test_buildvector_v32i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
;
; AVX2-32-LABEL: test_buildvector_v32i8:
; AVX2-32: # %bb.0:
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-32-NEXT: vpinsrb $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $4, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $5, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $6, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $7, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $8, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $9, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $10, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $11, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $12, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $13, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $14, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vpinsrb $15, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-32-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm2
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm2, %ymm2
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm1, %ymm1
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX2-32-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-32-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm3
+; AVX2-32-NEXT: vmovdqu {{[0-9]+}}(%esp), %ymm4
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm4, %ymm4
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]
+; AVX2-32-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-32-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
+; AVX2-32-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-32-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-32-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-32-NEXT: retl
;
; AVX2-64-LABEL: test_buildvector_v32i8:
diff --git a/llvm/test/CodeGen/X86/build-vector-512.ll b/llvm/test/CodeGen/X86/build-vector-512.ll
index ad4f76347f1c2..d8e9c47607f3d 100644
--- a/llvm/test/CodeGen/X86/build-vector-512.ll
+++ b/llvm/test/CodeGen/X86/build-vector-512.ll
@@ -222,43 +222,89 @@ define <64 x i8> @test_buildvector_v64i8(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4,
; AVX-32-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX-32-NEXT: retl
;
-; AVX-64-LABEL: test_buildvector_v64i8:
-; AVX-64: # %bb.0:
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
-; AVX-64-NEXT: vpmovqb %zmm1, %xmm1
-; AVX-64-NEXT: vpmovqb %zmm3, %xmm3
-; AVX-64-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX-64-NEXT: vpmovqb %zmm0, %xmm0
-; AVX-64-NEXT: vpmovqb %zmm2, %xmm2
-; AVX-64-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
-; AVX-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
-; AVX-64-NEXT: vpmovqb %zmm2, %xmm2
-; AVX-64-NEXT: vpmovqb %zmm1, %xmm1
-; AVX-64-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-64-NEXT: vmovd %edi, %xmm2
-; AVX-64-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
-; AVX-64-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX-64-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX-64-NEXT: retq
+; AVX512F-64-LABEL: test_buildvector_v64i8:
+; AVX512F-64: # %bb.0:
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; AVX512F-64-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512F-64-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512F-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-64-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-64-NEXT: vpmovqw %zmm2, %xmm1
+; AVX512F-64-NEXT: vpmovqw %zmm3, %xmm2
+; AVX512F-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-64-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-64-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512F-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-64-NEXT: vmovd %edi, %xmm1
+; AVX512F-64-NEXT: vpinsrb $1, %esi, %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $3, %ecx, %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $4, %r8d, %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $5, %r9d, %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm1, %xmm1
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512F-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; AVX512F-64-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512F-64-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512F-64-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512F-64-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512F-64-NEXT: vpmovdb %zmm2, %xmm2
+; AVX512F-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-64-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-64-NEXT: retq
+;
+; AVX512BW-64-LABEL: test_buildvector_v64i8:
+; AVX512BW-64: # %bb.0:
+; AVX512BW-64-NEXT: vmovd %edi, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $1, %esi, %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $4, %r8d, %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $5, %r9d, %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512BW-64-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512BW-64-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512BW-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-64-NEXT: vpmovwb %zmm1, %ymm1
+; AVX512BW-64-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm3
+; AVX512BW-64-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm4
+; AVX512BW-64-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512BW-64-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512BW-64-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
+; AVX512BW-64-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512BW-64-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512BW-64-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512BW-64-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512BW-64-NEXT: vpmovwb %zmm1, %ymm1
+; AVX512BW-64-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-64-NEXT: retq
%ins0 = insertelement <64 x i8> undef, i8 %a0, i32 0
%ins1 = insertelement <64 x i8> %ins0, i8 %a1, i32 1
%ins2 = insertelement <64 x i8> %ins1, i8 %a2, i32 2
diff --git a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
index befa448ab408a..db530a2618aa7 100644
--- a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
+++ b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
@@ -25,15 +25,28 @@ define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
; AVX2-NEXT: vpinsrb $15, 120(%rdi), %xmm0, %xmm0
; AVX2-NEXT: retq
;
-; AVX512-LABEL: buildvec_v16i8_stride8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlq $56, 57(%rdi), %zmm0
-; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm1
-; AVX512-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: buildvec_v16i8_stride8:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512F-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT: vpsrlq $56, 57(%rdi), %zmm1
+; AVX512F-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: buildvec_v16i8_stride8:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512VL-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512VL-NEXT: vpsrlq $56, 57(%rdi), %zmm1
+; AVX512VL-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
%p1 = getelementptr inbounds i8, ptr %p, i64 8
%p2 = getelementptr inbounds i8, ptr %p, i64 16
@@ -88,53 +101,36 @@ define <16 x i8> @buildvec_v16i8_stride8(ptr %p) {
define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v16i8_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzbl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu (%rdi), %ymm0
+; AVX2-NEXT: vmovdqu 29(%rdi), %ymm1
+; AVX2-NEXT: vpsrld $24, %ymm1, %ymm1
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: buildvec_v16i8_stride4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: movzbl (%rdi), %eax
-; AVX512F-NEXT: vmovd %eax, %xmm0
-; AVX512F-NEXT: vpinsrb $1, 4(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $2, 8(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $3, 12(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $4, 16(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $5, 20(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $6, 24(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, 28(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, 32(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, 36(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, 40(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, 44(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, 48(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, 52(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, 56(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, 60(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vmovdqu (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqu 29(%rdi), %ymm1
+; AVX512F-NEXT: vpsrld $24, %ymm1, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: buildvec_v16i8_stride4:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpsrld $24, 29(%rdi), %ymm0
-; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512VL-NEXT: vmovdqu (%rdi), %ymm1
-; AVX512VL-NEXT: vpmovdb %ymm1, %xmm1
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovdqu (%rdi), %ymm0
+; AVX512VL-NEXT: vpsrld $24, 29(%rdi), %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
@@ -191,28 +187,25 @@ define <16 x i8> @buildvec_v16i8_stride4(ptr %p) {
define <8 x i16> @buildvec_v8i16_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v8i16_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzwl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrw $1, 4(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $2, 8(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $3, 12(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $4, 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $5, 20(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $6, 24(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $7, 28(%rdi), %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu (%rdi), %xmm0
+; AVX2-NEXT: vmovdqu 14(%rdi), %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: buildvec_v8i16_stride4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: movzwl (%rdi), %eax
-; AVX512F-NEXT: vmovd %eax, %xmm0
-; AVX512F-NEXT: vpinsrw $1, 4(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $2, 8(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $3, 12(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $4, 16(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $5, 20(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $6, 24(%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrw $7, 28(%rdi), %xmm0, %xmm0
+; AVX512F-NEXT: vmovdqu (%rdi), %xmm0
+; AVX512F-NEXT: vmovdqu 14(%rdi), %xmm1
+; AVX512F-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: buildvec_v8i16_stride4:
@@ -253,24 +246,25 @@ define <8 x i16> @buildvec_v8i16_stride4(ptr %p) {
define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
; AVX2-LABEL: buildvec_v16i16_stride8:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzwl 64(%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrw $1, 72(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $2, 80(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $3, 88(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $4, 96(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $5, 104(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $6, 112(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $7, 120(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: movzwl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpinsrw $1, 8(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $2, 16(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $3, 24(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $4, 32(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $5, 40(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $6, 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrw $7, 56(%rdi), %xmm1, %xmm1
+; AVX2-NEXT: vmovups 64(%rdi), %xmm0
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-NEXT: vmovdqu 90(%rdi), %ymm1
+; AVX2-NEXT: vpsrlq $48, %ymm1, %ymm1
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovups (%rdi), %xmm2
+; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],mem[0,2]
+; AVX2-NEXT: vmovdqu 26(%rdi), %ymm3
+; AVX2-NEXT: vpsrlq $48, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT: vpshufb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX2-NEXT: retq
;
@@ -336,41 +330,28 @@ define <16 x i16> @buildvec_v16i16_stride8(ptr %p) {
define <32 x i8> @buildvec_v32i8_stride4(ptr %p) {
; AVX2-LABEL: buildvec_v32i8_stride4:
; AVX2: # %bb.0:
-; AVX2-NEXT: movzbl 64(%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrb $1, 68(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $2, 72(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $3, 76(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $4, 80(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, 84(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, 88(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, 92(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, 96(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, 100(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, 104(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, 108(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, 112(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, 116(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, 120(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, 124(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: movzbl (%rdi), %eax
-; AVX2-NEXT: vmovd %eax, %xmm1
-; AVX2-NEXT: vpinsrb $1, 4(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $2, 8(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $3, 12(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $4, 16(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $5, 20(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $6, 24(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $7, 28(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $8, 32(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $9, 36(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $10, 40(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $11, 44(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $12, 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $13, 52(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $14, 56(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vpinsrb $15, 60(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: vmovdqu (%rdi), %ymm0
+; AVX2-NEXT: vmovdqu 29(%rdi), %ymm1
+; AVX2-NEXT: vmovdqu 64(%rdi), %ymm2
+; AVX2-NEXT: vmovdqu 93(%rdi), %ymm3
+; AVX2-NEXT: vpsrld $24, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsrld $24, %ymm1, %ymm1
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT: vpackusdw %xmm5, %xmm1, %xmm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: buildvec_v32i8_stride4:
@@ -706,20 +687,40 @@ define <32 x i8> @buildvec_v32i8_stride8(ptr %p) {
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX2-NEXT: retq
;
-; AVX512-LABEL: buildvec_v32i8_stride8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1
-; AVX512-NEXT: vmovdqu64 128(%rdi), %zmm2
-; AVX512-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512-NEXT: vpsrlq $56, 185(%rdi), %zmm3
-; AVX512-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512-NEXT: retq
+; AVX512F-LABEL: buildvec_v32i8_stride8:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqu64 128(%rdi), %zmm2
+; AVX512F-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vpmovqw %zmm2, %xmm1
+; AVX512F-NEXT: vpsrlq $56, 185(%rdi), %zmm2
+; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: buildvec_v32i8_stride8:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VL-NEXT: vmovdqu64 128(%rdi), %zmm2
+; AVX512VL-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512VL-NEXT: vpsrlq $56, 185(%rdi), %zmm3
+; AVX512VL-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512VL-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512VL-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 0
%p1 = getelementptr inbounds i8, ptr %p, i64 8
%p2 = getelementptr inbounds i8, ptr %p, i64 16
@@ -1055,16 +1056,30 @@ define <16 x i8> @buildvec_v16i8_stride8_reverse(ptr %p) {
; AVX2-NEXT: vpinsrb $15, (%rdi), %xmm0, %xmm0
; AVX2-NEXT: retq
;
-; AVX512-LABEL: buildvec_v16i8_stride8_reverse:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlq $56, 57(%rdi), %zmm0
-; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vmovdqu64 (%rdi), %zmm1
-; AVX512-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: buildvec_v16i8_stride8_reverse:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512F-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT: vpsrlq $56, 57(%rdi), %zmm1
+; AVX512F-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: buildvec_v16i8_stride8_reverse:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
+; AVX512VL-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512VL-NEXT: vpsrlq $56, 57(%rdi), %zmm1
+; AVX512VL-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
%p0 = getelementptr inbounds i8, ptr %p, i64 120
%p1 = getelementptr inbounds i8, ptr %p, i64 112
%p2 = getelementptr inbounds i8, ptr %p, i64 104
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 3b58bbd719d79..2ddd6676249f9 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -2664,21 +2664,21 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: subq $192, %rsp
; AVX512F-NEXT: vmovdqu64 352(%rbp), %zmm2
; AVX512F-NEXT: vmovdqu64 416(%rbp), %zmm3
-; AVX512F-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512F-NEXT: vpmovqw %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm5 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512F-NEXT: vmovdqu64 224(%rbp), %zmm2
; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm4
-; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
-; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm2[0],xmm4[0]
+; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512F-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm4
; AVX512F-NEXT: vmovdqu64 96(%rbp), %zmm2
; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm6
-; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512F-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm6[0]
+; AVX512F-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512F-NEXT: vpmovqw %zmm6, %xmm6
+; AVX512F-NEXT: vinserti128 $1, %xmm6, %ymm2, %ymm2
; AVX512F-NEXT: vmovd %edi, %xmm6
; AVX512F-NEXT: vpinsrb $1, %esi, %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $2, %edx, %xmm6, %xmm6
@@ -2695,7 +2695,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm6, %xmm6
; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm6, %xmm6
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm7 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k2
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
@@ -2711,7 +2711,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
; AVX512F-NEXT: vpmovdb %zmm5, (%rsp)
; AVX512F-NEXT: vptestmd %zmm3, %zmm7, %k2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
; AVX512F-NEXT: vpextrd $1, %xmm6, %eax
; AVX512F-NEXT: vmovd %xmm6, %ecx
; AVX512F-NEXT: addl %eax, %ecx
@@ -4415,23 +4415,23 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: subq $576, %rsp # imm = 0x240
; AVX512F-NEXT: vmovdqu64 352(%rbp), %zmm4
; AVX512F-NEXT: vmovdqu64 416(%rbp), %zmm5
-; AVX512F-NEXT: vpmovqb %zmm5, %xmm5
-; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512F-NEXT: vpmovqw %zmm5, %xmm5
+; AVX512F-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k1
; AVX512F-NEXT: vmovdqu64 224(%rbp), %zmm4
; AVX512F-NEXT: vmovdqu64 288(%rbp), %zmm6
-; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512F-NEXT: vpmovqb %zmm4, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
+; AVX512F-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512F-NEXT: vpmovqw %zmm6, %xmm6
+; AVX512F-NEXT: vinserti128 $1, %xmm6, %ymm4, %ymm4
; AVX512F-NEXT: vmovdqu64 96(%rbp), %zmm6
; AVX512F-NEXT: vmovdqu64 160(%rbp), %zmm7
-; AVX512F-NEXT: vpmovqb %zmm7, %xmm7
-; AVX512F-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0]
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vpmovqw %zmm6, %xmm6
+; AVX512F-NEXT: vpmovqw %zmm7, %xmm7
+; AVX512F-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm6
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm6 = ymm6[0],zero,ymm6[1],zero,ymm6[2],zero,ymm6[3],zero,ymm6[4],zero,ymm6[5],zero,ymm6[6],zero,ymm6[7],zero,ymm6[8],zero,ymm6[9],zero,ymm6[10],zero,ymm6[11],zero,ymm6[12],zero,ymm6[13],zero,ymm6[14],zero,ymm6[15],zero
; AVX512F-NEXT: vptestmd %zmm5, %zmm6, %k2
; AVX512F-NEXT: vmovd %edi, %xmm6
; AVX512F-NEXT: vpinsrb $1, %esi, %xmm6, %xmm6
@@ -4461,7 +4461,7 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpaddd %xmm7, %xmm6, %xmm6
; AVX512F-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
; AVX512F-NEXT: vpaddd %xmm7, %xmm6, %xmm6
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
; AVX512F-NEXT: vpextrd $1, %xmm6, %eax
; AVX512F-NEXT: vmovd %xmm6, %ecx
; AVX512F-NEXT: addl %eax, %ecx
More information about the llvm-commits
mailing list