[llvm] [X86] lowerShuffleAsVTRUNC - use combineConcatVectorOps to catch more "cheap" concats (PR #145876)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 26 04:58:04 PDT 2025
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/145876
None
>From 9d56e2858350e58adf0e1ae13962b9cfdcca65b5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Thu, 26 Jun 2025 12:56:41 +0100
Subject: [PATCH] [X86] lowerShuffleAsVTRUNC - use combineConcatVectorOps to
catch more "cheap" concats
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 42 +-
.../vector-interleaved-load-i16-stride-4.ll | 1462 ++--
.../vector-interleaved-load-i8-stride-8.ll | 6448 ++++++++---------
3 files changed, 3687 insertions(+), 4265 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7c26dd6e2dc2f..307a237e2955c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10312,6 +10312,11 @@ static bool isNonZeroElementsInOrder(const APInt &Zeroable,
return true;
}
+static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
+ ArrayRef<SDValue> Ops, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget,
+ unsigned Depth = 0);
+
/// Try to lower a shuffle with a single PSHUFB of V1 or V2.
static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
ArrayRef<int> Mask, SDValue V1,
@@ -10692,7 +10697,8 @@ static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
SelectionDAG &DAG) {
assert((VT.is128BitVector() || VT.is256BitVector()) &&
"Unexpected VTRUNC type");
- if (!Subtarget.hasAVX512())
+ if (!Subtarget.hasAVX512() ||
+ (VT.is256BitVector() && !Subtarget.useAVX512Regs()))
return SDValue();
unsigned NumElts = VT.getVectorNumElements();
@@ -10721,30 +10727,19 @@ static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
bool UndefUppers =
UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
+ // As we're using both sources then we need to concat them together
+ // and truncate from the double-sized src.
+ MVT ConcatVT = VT.getDoubleNumVectorElementsVT();
+
// For offset truncations, ensure that the concat is cheap.
- if (Offset) {
- auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
- if (Lo.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
- Hi.getOpcode() == ISD::EXTRACT_SUBVECTOR)
- return Lo.getOperand(0) == Hi.getOperand(0);
- if (ISD::isNormalLoad(Lo.getNode()) &&
- ISD::isNormalLoad(Hi.getNode())) {
- auto *LDLo = cast<LoadSDNode>(Lo);
- auto *LDHi = cast<LoadSDNode>(Hi);
- return DAG.areNonVolatileConsecutiveLoads(
- LDHi, LDLo, Lo.getValueType().getStoreSize(), 1);
- }
- return false;
- };
- if (!IsCheapConcat(peekThroughBitcasts(V1), peekThroughBitcasts(V2)))
+ SDValue Src =
+ combineConcatVectorOps(DL, ConcatVT, {V1, V2}, DAG, Subtarget);
+ if (!Src) {
+ if (Offset)
continue;
+ Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
}
- // As we're using both sources then we need to concat them together
- // and truncate from the double-sized src.
- MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
- SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
-
MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
Src = DAG.getBitcast(SrcVT, Src);
@@ -42183,11 +42178,6 @@ static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
return SDValue();
}
-static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
- ArrayRef<SDValue> Ops, SelectionDAG &DAG,
- const X86Subtarget &Subtarget,
- unsigned Depth = 0);
-
/// Try to combine x86 target specific shuffles.
static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
SelectionDAG &DAG,
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
index ae4f85ce42a19..01aacc1e06258 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
@@ -1297,68 +1297,59 @@ define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512-LABEL: load_i16_stride4_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa 64(%rdi), %ymm0
-; AVX512-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512-NEXT: vmovdqa 64(%rdi), %ymm1
+; AVX512-NEXT: vpmovqw %ymm1, %xmm0
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 96(%rdi), %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa 112(%rdi), %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa 96(%rdi), %xmm5
+; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vmovdqa64 (%rdi), %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpsrlq $16, %ymm1, %ymm7
+; AVX512-NEXT: vpmovqw %ymm7, %xmm7
+; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vmovdqa64 (%rdi), %zmm3
-; AVX512-NEXT: vpmovqw %zmm3, %xmm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa 64(%rdi), %xmm6
-; AVX512-NEXT: vmovdqa 80(%rdi), %xmm7
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm4, %zmm6
+; AVX512-NEXT: vpmovqw %zmm6, %xmm6
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm6
+; AVX512-NEXT: vpmovqw %ymm6, %xmm6
+; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm4, %zmm7
+; AVX512-NEXT: vpmovqw %zmm7, %xmm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512-NEXT: vpsrlq $48, %ymm1, %ymm1
+; AVX512-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm3, %zmm5
-; AVX512-NEXT: vpmovqw %zmm5, %xmm5
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm3, %zmm8
-; AVX512-NEXT: vpmovqw %zmm8, %xmm8
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vpsrlq $48, %zmm3, %zmm3
-; AVX512-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT: vpsrlq $48, %zmm4, %zmm2
+; AVX512-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT: vmovdqa %ymm2, (%rdx)
-; AVX512-NEXT: vmovdqa %ymm5, (%rcx)
+; AVX512-NEXT: vmovdqa %ymm3, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512-NEXT: vmovdqa %ymm1, (%r8)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -1411,68 +1402,59 @@ define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512DQ-LABEL: load_i16_stride4_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm0
-; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm1
+; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm0
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm4
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm2
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm1, %ymm7
+; AVX512DQ-NEXT: vpmovqw %ymm7, %xmm7
+; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm3
-; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm6
-; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm7
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm4, %zmm6
+; AVX512DQ-NEXT: vpmovqw %zmm6, %xmm6
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm1, %ymm6
+; AVX512DQ-NEXT: vpmovqw %ymm6, %xmm6
+; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm4, %zmm7
+; AVX512DQ-NEXT: vpmovqw %zmm7, %xmm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm3, %zmm5
-; AVX512DQ-NEXT: vpmovqw %zmm5, %xmm5
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm3, %zmm8
-; AVX512DQ-NEXT: vpmovqw %zmm8, %xmm8
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm3, %zmm3
-; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm4, %zmm2
+; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %ymm2, (%rdx)
-; AVX512DQ-NEXT: vmovdqa %ymm5, (%rcx)
+; AVX512DQ-NEXT: vmovdqa %ymm3, (%rdx)
+; AVX512DQ-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%r8)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
@@ -2713,128 +2695,110 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm1
; AVX512-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512-NEXT: vpmovqw %ymm2, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm6
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm7
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT: vpmovqw %ymm2, %xmm3
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vpmovqw %zmm1, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vmovdqa 240(%rdi), %xmm5
+; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa 224(%rdi), %xmm7
+; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpmovqw %zmm1, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512-NEXT: vpmovqw %ymm3, %xmm3
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm10
-; AVX512-NEXT: vmovdqa 112(%rdi), %xmm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 96(%rdi), %xmm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX512-NEXT: vpmovqw %zmm0, %xmm11
-; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512-NEXT: vmovdqa 80(%rdi), %xmm11
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm13
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm14
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm13[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
+; AVX512-NEXT: vpmovqw %ymm3, %xmm9
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm1, %zmm9
-; AVX512-NEXT: vpmovqw %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-NEXT: vmovdqa 112(%rdi), %xmm10
+; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa 96(%rdi), %xmm13
+; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]
+; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]
+; AVX512-NEXT: vpmovqw %zmm0, %xmm12
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]
+; AVX512-NEXT: vpsrlq $16, %ymm2, %ymm9
+; AVX512-NEXT: vpmovqw %ymm9, %xmm9
+; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm12[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm1, %zmm8
+; AVX512-NEXT: vpmovqw %zmm8, %xmm8
+; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512-NEXT: vpsrlq $16, %ymm3, %ymm8
+; AVX512-NEXT: vpmovqw %ymm8, %xmm8
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
; AVX512-NEXT: vpsrlq $16, %zmm0, %zmm9
; AVX512-NEXT: vpmovqw %zmm9, %xmm9
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[0,1,2,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm6[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]
+; AVX512-NEXT: vpsrlq $32, %ymm2, %ymm8
+; AVX512-NEXT: vpmovqw %ymm8, %xmm8
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm12
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm14[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm13
-; AVX512-NEXT: vpmovqw %zmm13, %xmm13
-; AVX512-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm11[2,0,2,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm9
+; AVX512-NEXT: vpmovqw %zmm9, %xmm9
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT: vpsrlq $32, %ymm3, %ymm9
+; AVX512-NEXT: vpmovqw %ymm9, %xmm9
+; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm10[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
-; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm14
-; AVX512-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm13[0,1,2,3],zmm12[0,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
+; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm11
+; AVX512-NEXT: vpmovqw %zmm11, %xmm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]
+; AVX512-NEXT: vpsrlq $48, %ymm2, %ymm2
+; AVX512-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
; AVX512-NEXT: vpsrlq $48, %zmm1, %zmm1
; AVX512-NEXT: vpmovqw %zmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vpsrlq $48, %ymm3, %ymm2
+; AVX512-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
; AVX512-NEXT: vpsrlq $48, %zmm0, %zmm0
; AVX512-NEXT: vpmovqw %zmm0, %xmm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT: vmovdqa64 %zmm2, (%rsi)
-; AVX512-NEXT: vmovdqa64 %zmm5, (%rdx)
-; AVX512-NEXT: vmovdqa64 %zmm12, (%rcx)
+; AVX512-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512-NEXT: vmovdqa64 %zmm6, (%rdx)
+; AVX512-NEXT: vmovdqa64 %zmm8, (%rcx)
; AVX512-NEXT: vmovdqa64 %zmm0, (%r8)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -2924,128 +2888,110 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm1
; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm6
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm7
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm3
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm5
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm7
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm3
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm10
-; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm3
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm4
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm11
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm11
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm13
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm14
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm13[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
+; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm9
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm1, %zmm9
-; AVX512DQ-NEXT: vpmovqw %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm10
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm13
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]
+; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm12
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm2, %ymm9
+; AVX512DQ-NEXT: vpmovqw %ymm9, %xmm9
+; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm12[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm1, %zmm8
+; AVX512DQ-NEXT: vpmovqw %zmm8, %xmm8
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm3, %ymm8
+; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm8
; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
; AVX512DQ-NEXT: vpsrlq $16, %zmm0, %zmm9
; AVX512DQ-NEXT: vpmovqw %zmm9, %xmm9
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[0,1,2,3]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm6[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm2, %ymm8
+; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm7[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm12
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm14[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm1, %zmm13
-; AVX512DQ-NEXT: vpmovqw %zmm13, %xmm13
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm11[2,0,2,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm1, %zmm9
+; AVX512DQ-NEXT: vpmovqw %zmm9, %xmm9
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm3, %ymm9
+; AVX512DQ-NEXT: vpmovqw %ymm9, %xmm9
+; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm15 = xmm10[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm0, %zmm14
-; AVX512DQ-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm13[0,1,2,3],zmm12[0,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm0, %zmm11
+; AVX512DQ-NEXT: vpmovqw %zmm11, %xmm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
; AVX512DQ-NEXT: vpsrlq $48, %zmm1, %zmm1
; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm3, %ymm2
+; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
; AVX512DQ-NEXT: vpsrlq $48, %zmm0, %zmm0
; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqa64 %zmm2, (%rsi)
-; AVX512DQ-NEXT: vmovdqa64 %zmm5, (%rdx)
-; AVX512DQ-NEXT: vmovdqa64 %zmm12, (%rcx)
+; AVX512DQ-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %zmm8, (%rcx)
; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r8)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
@@ -5673,276 +5619,222 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512-LABEL: load_i16_stride4_vf64:
; AVX512: # %bb.0:
-; AVX512-NEXT: subq $200, %rsp
-; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm26
-; AVX512-NEXT: vmovdqa64 384(%rdi), %zmm27
-; AVX512-NEXT: vmovdqa64 (%rdi), %zmm28
-; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm29
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm22
+; AVX512-NEXT: vmovdqa64 384(%rdi), %zmm23
+; AVX512-NEXT: vmovdqa64 (%rdi), %zmm29
+; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm30
+; AVX512-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512-NEXT: vpmovqw %ymm5, %xmm0
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm14
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm13
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm13[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm24
+; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm26
+; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vpmovqw %zmm29, %xmm1
+; AVX512-NEXT: vpmovqw %zmm30, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512-NEXT: vmovdqa 64(%rdi), %ymm6
+; AVX512-NEXT: vpmovqw %ymm6, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vmovdqa 112(%rdi), %xmm12
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 96(%rdi), %xmm11
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT: vmovdqa64 112(%rdi), %xmm27
+; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa64 96(%rdi), %xmm28
+; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
+; AVX512-NEXT: vpmovqw %zmm29, %xmm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512-NEXT: vmovdqa 448(%rdi), %ymm7
+; AVX512-NEXT: vpmovqw %ymm7, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm8
+; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm16
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa64 480(%rdi), %xmm17
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT: vpmovqw %zmm23, %xmm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT: vmovdqa 320(%rdi), %ymm8
+; AVX512-NEXT: vpmovqw %ymm8, %xmm11
+; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm15
+; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm18
+; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm19
+; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT: vpmovqw %zmm22, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]
+; AVX512-NEXT: vpsrlq $16, %ymm5, %ymm2
+; AVX512-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpmovqw %zmm28, %xmm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
-; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm0
-; AVX512-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm30, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vpsrlq $16, %ymm6, %ymm4
+; AVX512-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm29, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]
+; AVX512-NEXT: vpsrlq $16, %ymm7, %ymm2
+; AVX512-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm24
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm24[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 480(%rdi), %xmm23
-; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm23[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vpmovqw %zmm27, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm23, %zmm1
+; AVX512-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpsrlq $16, %ymm8, %ymm1
; AVX512-NEXT: vpmovqw %ymm1, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm31
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm31[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm25
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm25[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vpmovqw %zmm26, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa64 320(%rdi), %xmm30
-; AVX512-NEXT: vmovdqa64 336(%rdi), %xmm17
-; AVX512-NEXT: vmovdqa64 448(%rdi), %xmm18
-; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm19
-; AVX512-NEXT: vmovdqa64 64(%rdi), %xmm20
-; AVX512-NEXT: vmovdqa64 80(%rdi), %xmm21
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm1
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm29, %zmm6
-; AVX512-NEXT: vpmovqw %zmm6, %xmm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm21[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm20[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm28, %zmm6
-; AVX512-NEXT: vpmovqw %zmm6, %xmm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT: vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm19[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm18[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm27, %zmm3
-; AVX512-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm17[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm30[0,2,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT: vpsrlq $16, %zmm22, %zmm2
+; AVX512-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512-NEXT: vpsrlq $32, %ymm5, %ymm0
+; AVX512-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm30, %zmm1
+; AVX512-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpsrlq $32, %ymm6, %ymm0
+; AVX512-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vpsrlq $16, %zmm26, %zmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm29, %zmm4
; AVX512-NEXT: vpmovqw %zmm4, %xmm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 %xmm2, %xmm22
-; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]
+; AVX512-NEXT: vpsrlq $32, %ymm7, %ymm1
+; AVX512-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm23, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vpsrlq $32, %ymm8, %ymm4
+; AVX512-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]
+; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT: vpsrlq $32, %zmm22, %zmm11
+; AVX512-NEXT: vpmovqw %zmm11, %xmm11
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]
+; AVX512-NEXT: vpsrlq $48, %ymm5, %ymm4
+; AVX512-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm29, %zmm1
-; AVX512-NEXT: vpmovqw %zmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vpsrlq $48, %zmm30, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT: vpsrlq $48, %ymm6, %ymm4
+; AVX512-NEXT: vpmovqw %ymm4, %xmm4
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm21[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 %xmm0, %xmm21
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm20[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 %xmm0, %xmm16
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm28, %zmm8
-; AVX512-NEXT: vpmovqw %zmm8, %xmm8
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm23[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm20 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm18[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm18 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm31[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm25[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm15[0,1,2,0,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm19 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm17[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm30[3,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
-; AVX512-NEXT: vinserti32x4 $1, %xmm20, %ymm0, %ymm14
-; AVX512-NEXT: vinserti32x4 $1, %xmm18, %ymm0, %ymm5
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm14[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm27, %zmm14
-; AVX512-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm14
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm14[6,7]
-; AVX512-NEXT: vpsrlq $32, %zmm26, %zmm14
-; AVX512-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm0[0,1,2,3],zmm5[0,1,2,3]
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm0
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm13[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vpsrlq $48, %zmm29, %zmm3
-; AVX512-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm3
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vmovdqa64 %xmm16, %xmm5
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT: vpsrlq $48, %zmm29, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512-NEXT: vpsrlq $48, %ymm7, %ymm3
+; AVX512-NEXT: vpmovqw %ymm3, %xmm3
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpsrlq $48, %zmm28, %zmm3
-; AVX512-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm0[0,1,2,3]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpsrlq $48, %zmm27, %zmm2
-; AVX512-NEXT: vpmovqw %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vpsrlq $48, %zmm26, %zmm3
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT: vpsrlq $48, %zmm23, %zmm3
; AVX512-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT: vmovaps %zmm2, 64(%rsi)
-; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT: vmovaps %zmm2, (%rsi)
-; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT: vmovaps %zmm2, 64(%rdx)
-; AVX512-NEXT: vmovups (%rsp), %zmm2 # 64-byte Reload
-; AVX512-NEXT: vmovaps %zmm2, (%rdx)
-; AVX512-NEXT: vmovdqa64 %zmm17, 64(%rcx)
-; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT: vmovaps %zmm2, (%rcx)
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vpsrlq $48, %ymm8, %ymm3
+; AVX512-NEXT: vpmovqw %ymm3, %xmm3
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpsrlq $48, %zmm22, %zmm4
+; AVX512-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
+; AVX512-NEXT: vmovdqa64 %zmm21, 64(%rsi)
+; AVX512-NEXT: vmovdqa64 %zmm20, (%rsi)
+; AVX512-NEXT: vmovdqa64 %zmm31, 64(%rdx)
+; AVX512-NEXT: vmovdqa64 %zmm25, (%rdx)
+; AVX512-NEXT: vmovdqa64 %zmm2, 64(%rcx)
+; AVX512-NEXT: vmovdqa64 %zmm24, (%rcx)
; AVX512-NEXT: vmovdqa64 %zmm1, 64(%r8)
; AVX512-NEXT: vmovdqa64 %zmm0, (%r8)
-; AVX512-NEXT: addq $200, %rsp
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -6096,276 +5988,222 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512DQ-LABEL: load_i16_stride4_vf64:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: subq $200, %rsp
-; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm26
-; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %zmm27
-; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm28
-; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm29
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm22
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %zmm23
+; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm29
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm30
+; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512DQ-NEXT: vpmovqw %ymm5, %xmm0
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm14
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm13
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm13[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm24
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm26
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm29, %xmm1
+; AVX512DQ-NEXT: vpmovqw %zmm30, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm6
+; AVX512DQ-NEXT: vpmovqw %ymm6, %xmm1
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm12
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm11
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT: vmovdqa64 112(%rdi), %xmm27
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa64 96(%rdi), %xmm28
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
+; AVX512DQ-NEXT: vpmovqw %zmm29, %xmm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm7
+; AVX512DQ-NEXT: vpmovqw %ymm7, %xmm0
+; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm8
+; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm16
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %xmm17
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT: vpmovqw %zmm23, %xmm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm8
+; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm11
+; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm15
+; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm18
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm19
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT: vpmovqw %zmm22, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm5, %ymm2
+; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm28, %xmm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm0
-; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm30, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm6, %ymm4
+; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm29, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm7, %ymm2
+; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm24
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm24[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %xmm23
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm23[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm15[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm27, %xmm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm23, %zmm1
+; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm8, %ymm1
; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm1
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm31
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm31[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm25
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm25[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vpmovqw %zmm26, %xmm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %xmm30
-; AVX512DQ-NEXT: vmovdqa64 336(%rdi), %xmm17
-; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %xmm18
-; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm19
-; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %xmm20
-; AVX512DQ-NEXT: vmovdqa64 80(%rdi), %xmm21
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm1
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm29, %zmm6
-; AVX512DQ-NEXT: vpmovqw %zmm6, %xmm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm21[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm20[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm28, %zmm6
-; AVX512DQ-NEXT: vpmovqw %zmm6, %xmm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm19[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm18[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm27, %zmm3
-; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm17[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm30[0,2,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm22, %zmm2
+; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm5, %ymm0
+; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm30, %zmm1
+; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm1
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm6, %ymm0
+; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm26, %zmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm29, %zmm4
; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm22
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm7, %ymm1
+; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm23, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm8, %ymm4
+; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm22, %zmm11
+; AVX512DQ-NEXT: vpmovqw %zmm11, %xmm11
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm5, %ymm4
+; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm29, %zmm1
-; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm30, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm6, %ymm4
+; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm4
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm21[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm21
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm20[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm16
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm28, %zmm8
-; AVX512DQ-NEXT: vpmovqw %zmm8, %xmm8
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm23[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm20 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm18[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm18 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm31[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm25[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm15[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm19 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm17[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm30[3,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
-; AVX512DQ-NEXT: vinserti32x4 $1, %xmm20, %ymm0, %ymm14
-; AVX512DQ-NEXT: vinserti32x4 $1, %xmm18, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm14[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm27, %zmm14
-; AVX512DQ-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm14
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm14[6,7]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm26, %zmm14
-; AVX512DQ-NEXT: vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm17 = zmm0[0,1,2,3],zmm5[0,1,2,3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm0
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm13[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm29, %zmm3
-; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm3
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm5
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm29, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm7, %ymm3
+; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm3
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm28, %zmm3
-; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm0[0,1,2,3]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm27, %zmm2
-; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm26, %zmm3
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm23, %zmm3
; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT: vmovaps %zmm2, 64(%rsi)
-; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT: vmovaps %zmm2, (%rsi)
-; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT: vmovaps %zmm2, 64(%rdx)
-; AVX512DQ-NEXT: vmovups (%rsp), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT: vmovaps %zmm2, (%rdx)
-; AVX512DQ-NEXT: vmovdqa64 %zmm17, 64(%rcx)
-; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT: vmovaps %zmm2, (%rcx)
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm8, %ymm3
+; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm3
+; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm22, %zmm4
+; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
+; AVX512DQ-NEXT: vmovdqa64 %zmm21, 64(%rsi)
+; AVX512DQ-NEXT: vmovdqa64 %zmm20, (%rsi)
+; AVX512DQ-NEXT: vmovdqa64 %zmm31, 64(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %zmm25, (%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %zmm2, 64(%rcx)
+; AVX512DQ-NEXT: vmovdqa64 %zmm24, (%rcx)
; AVX512DQ-NEXT: vmovdqa64 %zmm1, 64(%r8)
; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r8)
-; AVX512DQ-NEXT: addq $200, %rsp
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
index 99932c0026b23..6770fb6660606 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
@@ -5975,343 +5975,275 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-LABEL: load_i8_stride8_vf32:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm12
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm1
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm7
-; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm9
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512-NEXT: vpshufb %xmm4, %xmm9, %xmm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm8
-; AVX512-NEXT: vpshufb %xmm4, %xmm8, %xmm6
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm2
+; AVX512-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
+; AVX512-NEXT: vpshufb %xmm7, %xmm1, %xmm3
+; AVX512-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512-NEXT: vpshufb %xmm7, %xmm5, %xmm4
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT: vmovdqa 208(%rdi), %xmm6
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
+; AVX512-NEXT: vpshufb %xmm8, %xmm6, %xmm4
+; AVX512-NEXT: vmovdqa 192(%rdi), %xmm10
+; AVX512-NEXT: vpshufb %xmm8, %xmm10, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm4
; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vmovdqa 112(%rdi), %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm6
-; AVX512-NEXT: vmovdqa 96(%rdi), %xmm1
-; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm18
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX512-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512-NEXT: vpshufb %xmm4, %xmm1, %xmm11
-; AVX512-NEXT: vmovdqa64 %xmm1, %xmm21
-; AVX512-NEXT: vmovdqa 64(%rdi), %xmm3
-; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
-; AVX512-NEXT: vpmovqb %zmm16, %xmm10
-; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpmovqb %ymm2, %xmm9
+; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vmovdqa 112(%rdi), %xmm3
+; AVX512-NEXT: vpshufb %xmm7, %xmm3, %xmm9
+; AVX512-NEXT: vmovdqa 96(%rdi), %xmm4
+; AVX512-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512-NEXT: vmovdqa 80(%rdi), %xmm7
+; AVX512-NEXT: vpshufb %xmm8, %xmm7, %xmm13
+; AVX512-NEXT: vmovdqa 64(%rdi), %xmm9
+; AVX512-NEXT: vpshufb %xmm8, %xmm9, %xmm8
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512-NEXT: vpmovqb %zmm16, %xmm12
+; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm17
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm10
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm6
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm12, %xmm19
-; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm12
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vmovdqa 160(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa 176(%rdi), %xmm12
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512-NEXT: vpshufb %xmm8, %xmm1, %xmm13
+; AVX512-NEXT: vpshufb %xmm8, %xmm5, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512-NEXT: vpshufb %xmm14, %xmm9, %xmm12
-; AVX512-NEXT: vmovdqa64 %xmm9, %xmm22
-; AVX512-NEXT: vpshufb %xmm14, %xmm8, %xmm13
-; AVX512-NEXT: vmovdqa64 %xmm8, %xmm20
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm12, %xmm6, %xmm13
-; AVX512-NEXT: vpshufb %xmm12, %xmm10, %xmm12
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm8
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm13
-; AVX512-NEXT: vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm0
-; AVX512-NEXT: vpshufb %xmm1, %xmm8, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm1
+; AVX512-NEXT: vpshufb %xmm14, %xmm6, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm10, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm24
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm5
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm4
-; AVX512-NEXT: vpshufb %xmm14, %xmm4, %xmm2
-; AVX512-NEXT: vpshufb %xmm14, %xmm3, %xmm4
-; AVX512-NEXT: vmovdqa %xmm3, %xmm11
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $8, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm13[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm13 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm13, %xmm11, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT: vpsrlq $8, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT: vpshufb %xmm8, %xmm3, %xmm13
+; AVX512-NEXT: vpshufb %xmm8, %xmm4, %xmm8
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512-NEXT: vpshufb %xmm14, %xmm9, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512-NEXT: vpsrlq $8, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm18
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512-NEXT: vmovdqa64 %xmm19, %xmm3
-; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm1
-; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm2
-; AVX512-NEXT: vmovdqa %xmm7, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm8
-; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm7
-; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm14
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm4, %xmm6, %xmm14
-; AVX512-NEXT: vmovdqa %xmm6, %xmm12
-; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm6
-; AVX512-NEXT: vpshufb %xmm14, %xmm6, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm13 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm20
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm6
-; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $16, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vpsrlq $16, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $16, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm3, %xmm25
-; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm14
-; AVX512-NEXT: vmovdqa64 %xmm7, %xmm22
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm14
-; AVX512-NEXT: vmovdqa64 %xmm12, %xmm27
-; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm12
-; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm13 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm5
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm15
-; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm0
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm2
-; AVX512-NEXT: vmovdqa %xmm11, %xmm6
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $24, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT: vpsrlq $24, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $24, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm20
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm11
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm1
-; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm2
-; AVX512-NEXT: vmovdqa %xmm9, %xmm7
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm9
-; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm14
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm0
-; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm14
-; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm10, %xmm22
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm14, %xmm13, %xmm0
-; AVX512-NEXT: vmovdqa64 %xmm13, %xmm25
-; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm14
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm1
-; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm10
-; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-NEXT: vmovdqa64 %xmm6, %xmm23
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $32, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm13 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpsrlq $32, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $32, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm21
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
-; AVX512-NEXT: vpshufb %xmm5, %xmm11, %xmm1
-; AVX512-NEXT: vpshufb %xmm5, %xmm7, %xmm2
-; AVX512-NEXT: vmovdqa %xmm7, %xmm13
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm4
-; AVX512-NEXT: vmovdqa %xmm9, %xmm14
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm6
-; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm7
-; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm9
-; AVX512-NEXT: vpshufb %xmm4, %xmm9, %xmm0
-; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm8
-; AVX512-NEXT: vpshufb %xmm5, %xmm8, %xmm1
-; AVX512-NEXT: vpshufb %xmm5, %xmm15, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm10, %xmm24
-; AVX512-NEXT: vmovdqa64 %xmm23, %xmm4
-; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $40, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm15
+; AVX512-NEXT: vpshufb %xmm13, %xmm10, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm14 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT: vpsrlq $40, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $40, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm22
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512-NEXT: vpshufb %xmm10, %xmm11, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm11, %xmm23
-; AVX512-NEXT: vpshufb %xmm10, %xmm13, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm11
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm5
-; AVX512-NEXT: vmovdqa64 %xmm14, %xmm25
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
+; AVX512-NEXT: vpshufb %xmm14, %xmm6, %xmm15
+; AVX512-NEXT: vpshufb %xmm14, %xmm10, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm13 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm13, %xmm12, %xmm15
+; AVX512-NEXT: vpshufb %xmm13, %xmm11, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT: vpsrlq $48, %ymm2, %ymm15
+; AVX512-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm13
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512-NEXT: vpshufb %xmm14, %xmm9, %xmm14
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $48, %zmm16, %zmm13
+; AVX512-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
+; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm5 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
+; AVX512-NEXT: vpshufb %xmm5, %xmm6, %xmm6
+; AVX512-NEXT: vpshufb %xmm5, %xmm10, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm5, %xmm9, %xmm0
-; AVX512-NEXT: vmovdqa %xmm9, %xmm14
-; AVX512-NEXT: vpshufb %xmm5, %xmm12, %xmm5
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vpshufb %xmm10, %xmm8, %xmm1
-; AVX512-NEXT: vmovdqa %xmm8, %xmm9
-; AVX512-NEXT: vpshufb %xmm10, %xmm15, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm10
-; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT: vmovdqa %xmm4, %xmm8
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $48, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512-NEXT: vmovdqa64 %xmm23, %xmm2
-; AVX512-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm5
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5,6],ymm1[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm6 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm6, %xmm12, %xmm10
+; AVX512-NEXT: vpshufb %xmm6, %xmm11, %xmm6
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT: vpsrlq $56, %ymm2, %ymm2
+; AVX512-NEXT: vpmovqb %ymm2, %xmm2
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm4, %xmm6, %xmm5
-; AVX512-NEXT: vpshufb %xmm4, %xmm7, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm5, %xmm14, %xmm7
-; AVX512-NEXT: vpshufb %xmm5, %xmm12, %xmm5
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm4
-; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm4
-; AVX512-NEXT: vpshufb %xmm3, %xmm8, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $56, %zmm16, %zmm3
-; AVX512-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5],ymm2[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm2
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512-NEXT: vpshufb %xmm5, %xmm7, %xmm2
+; AVX512-NEXT: vpshufb %xmm5, %xmm9, %xmm3
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $56, %zmm16, %zmm2
+; AVX512-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm17, (%rsi)
; AVX512-NEXT: vmovdqa64 %ymm18, (%rdx)
; AVX512-NEXT: vmovdqa64 %ymm19, (%rcx)
@@ -6320,9 +6252,9 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: vmovdqa64 %ymm22, (%rax)
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT: vmovdqa %ymm0, (%rax)
+; AVX512-NEXT: vmovdqa %ymm8, (%rax)
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT: vmovdqa %ymm1, (%rax)
+; AVX512-NEXT: vmovdqa %ymm0, (%rax)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -6514,343 +6446,275 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-LABEL: load_i8_stride8_vf32:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm12
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm1
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm9
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm9, %xmm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm8, %xmm6
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm2
+; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm7 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
+; AVX512DQ-NEXT: vpshufb %xmm7, %xmm1, %xmm3
+; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm7, %xmm5, %xmm4
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm6
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm6, %xmm4
+; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm10, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm4
; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512DQ-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm6
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm18
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm1, %xmm11
-; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm21
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
-; AVX512DQ-NEXT: vpmovqb %zmm16, %xmm10
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpmovqb %ymm2, %xmm9
+; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm7, %xmm3, %xmm9
+; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm7, %xmm13
+; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm9, %xmm8
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512DQ-NEXT: vpmovqb %zmm16, %xmm12
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm17
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm10
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm6
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm19
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm12
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm12
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm1, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm5, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm9, %xmm12
-; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm22
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm8, %xmm13
-; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm20
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm12, %xmm6, %xmm13
-; AVX512DQ-NEXT: vpshufb %xmm12, %xmm10, %xmm12
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm8
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm13
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm8, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm1
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm6, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm10, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm24
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm4, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm3, %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm3, %xmm11
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $8, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm13[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm13 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm11, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT: vpsrlq $8, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm3, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm8, %xmm4, %xmm8
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm9, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512DQ-NEXT: vpsrlq $8, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm18
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm7, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm14
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm6, %xmm14
-; AVX512DQ-NEXT: vmovdqa %xmm6, %xmm12
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm6, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm13 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm0
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm20
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $16, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm25
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm14
-; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm22
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm14
-; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm27
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm12
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm13 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm15
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm0
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm11, %xmm6
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $24, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT: vpsrlq $24, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $24, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm7
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm14
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm0, %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm22
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm13, %xmm0
-; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm25
-; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm14
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm10
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm23
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm13 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpsrlq $32, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $32, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm21
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm11, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm7, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm7, %xmm13
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm14
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm9, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm8, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm15, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm24
-; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $40, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm6, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm10, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm14 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT: vpsrlq $40, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $40, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm22
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512DQ-NEXT: vpshufb %xmm10, %xmm11, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm23
-; AVX512DQ-NEXT: vpshufb %xmm10, %xmm13, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm5
-; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm25
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm6, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm10, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm13 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm12, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm13, %xmm11, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT: vpsrlq $48, %ymm2, %ymm15
+; AVX512DQ-NEXT: vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm14, %xmm9, %xmm14
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm16, %zmm13
+; AVX512DQ-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm5 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm6, %xmm6
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm10, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm9, %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm9, %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm12, %xmm5
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm10, %xmm8, %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm8, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm10, %xmm15, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm10
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm4, %xmm8
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm6 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm6, %xmm12, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm6, %xmm11, %xmm6
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpsrlq $56, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpmovqb %ymm2, %xmm2
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm6, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm7, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm14, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm5, %xmm12, %xmm5
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm8, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $56, %zmm16, %zmm3
-; AVX512DQ-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5],ymm2[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm2
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm7, %xmm2
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm9, %xmm3
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $56, %zmm16, %zmm2
+; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm17, (%rsi)
; AVX512DQ-NEXT: vmovdqa64 %ymm18, (%rdx)
; AVX512DQ-NEXT: vmovdqa64 %ymm19, (%rcx)
@@ -6859,9 +6723,9 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-NEXT: vmovdqa64 %ymm22, (%rax)
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT: vmovdqa %ymm0, (%rax)
+; AVX512DQ-NEXT: vmovdqa %ymm8, (%rax)
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT: vmovdqa %ymm1, (%rax)
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rax)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -7055,300 +6919,284 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT: vmovdqa 240(%rdi), %xmm2
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm2, %xmm1
-; AVX512BW-NEXT: vmovdqa 224(%rdi), %xmm6
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm6, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm7
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm7, %xmm3
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm9
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm9, %xmm8
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512BW-NEXT: vpmovqb %ymm3, %xmm3
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm17
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT: vpmovqb %zmm1, %xmm2
+; AVX512BW-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT: vpshufb %xmm7, %xmm1, %xmm3
+; AVX512BW-NEXT: vmovdqa 224(%rdi), %xmm4
+; AVX512BW-NEXT: vpshufb %xmm7, %xmm4, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm1, %xmm8
-; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm3
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
-; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm4
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm4, %xmm11
-; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm8, %xmm5
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
-; AVX512BW-NEXT: vpmovqb %zmm16, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm20
-; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm19
-; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm11
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm12
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm13
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm5, %xmm6
+; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm9
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm9, %xmm10
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5,6],ymm3[7]
+; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm6
+; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT: vpmovqb %ymm2, %xmm10
+; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3,4],ymm6[5],ymm10[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5],ymm3[6,7]
+; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm3
+; AVX512BW-NEXT: vpshufb %xmm7, %xmm3, %xmm10
+; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm6
+; AVX512BW-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]
+; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm7
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm7, %xmm13
+; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm10
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm10, %xmm8
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512BW-NEXT: vpmovqb %zmm17, %xmm12
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm11
+; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm12
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm1, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm4, %xmm14
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm5, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm16
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $8, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm21
+; AVX512BW-NEXT: vpsrlq $8, %ymm2, %ymm16
+; AVX512BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm3, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm10, %xmm14
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512BW-NEXT: vpsrlq $8, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm15
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT: vpsrlq $16, %ymm2, %ymm15
+; AVX512BW-NEXT: vpmovqb %ymm15, %xmm15
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $16, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $16, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm20
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm15
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT: vpsrlq $24, %ymm2, %ymm15
+; AVX512BW-NEXT: vpmovqb %ymm15, %xmm15
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $24, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm23
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $24, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm21
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm16
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm13, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm12, %xmm16
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm15
+; AVX512BW-NEXT: vpsrlq $32, %ymm2, %ymm16
+; AVX512BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm15
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $32, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm24
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $32, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm22
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm7, %xmm15
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm5, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm16
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm14
+; AVX512BW-NEXT: vpsrlq $40, %ymm2, %ymm16
+; AVX512BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm14
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $40, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm25
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $40, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm2, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm14
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm5, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm9, %xmm16
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm5
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $48, %zmm16, %zmm10
-; AVX512BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT: vpsrlq $48, %ymm2, %ymm16
+; AVX512BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm14, %xmm10, %xmm14
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $48, %zmm17, %zmm13
+; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm9
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm6, %xmm6
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512BW-NEXT: vpshufb %xmm5, %xmm12, %xmm9
+; AVX512BW-NEXT: vpshufb %xmm5, %xmm11, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpsrlq $56, %ymm2, %ymm2
+; AVX512BW-NEXT: vpmovqb %ymm2, %xmm2
; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm6, %xmm7, %xmm7
-; AVX512BW-NEXT: vpshufb %xmm6, %xmm9, %xmm9
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm7, %xmm13, %xmm9
-; AVX512BW-NEXT: vpshufb %xmm7, %xmm12, %xmm7
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm9, %xmm11, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm9, %xmm19, %xmm9
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm1, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm5, %xmm3, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm6, %xmm4, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm6, %xmm8, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT: vpsrlq $56, %zmm16, %zmm3
-; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa64 %ymm20, (%rsi)
-; AVX512BW-NEXT: vmovdqa64 %ymm21, (%rdx)
-; AVX512BW-NEXT: vmovdqa64 %ymm22, (%rcx)
-; AVX512BW-NEXT: vmovdqa64 %ymm23, (%r8)
-; AVX512BW-NEXT: vmovdqa64 %ymm24, (%r9)
-; AVX512BW-NEXT: vmovdqa64 %ymm25, (%r11)
-; AVX512BW-NEXT: vmovdqa %ymm0, (%r10)
-; AVX512BW-NEXT: vmovdqa %ymm1, (%rax)
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm3, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm7, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm10, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $56, %zmm17, %zmm2
+; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT: vmovdqa64 %ymm18, (%rsi)
+; AVX512BW-NEXT: vmovdqa64 %ymm19, (%rdx)
+; AVX512BW-NEXT: vmovdqa64 %ymm20, (%rcx)
+; AVX512BW-NEXT: vmovdqa64 %ymm21, (%r8)
+; AVX512BW-NEXT: vmovdqa64 %ymm22, (%r9)
+; AVX512BW-NEXT: vmovdqa64 %ymm23, (%r11)
+; AVX512BW-NEXT: vmovdqa %ymm8, (%r10)
+; AVX512BW-NEXT: vmovdqa %ymm0, (%rax)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -7525,300 +7373,284 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa 240(%rdi), %xmm2
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm2, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %xmm6
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm6, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm7
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm7, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm9
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm9, %xmm8
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vpmovqb %ymm3, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm17
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm1, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm4, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm1, %xmm8
-; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
-; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm4, %xmm11
-; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm8, %xmm5
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
-; AVX512DQ-BW-NEXT: vpmovqb %zmm16, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm19
-; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm11
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm12
-; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm13
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm5, %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm9
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm9, %xmm10
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vpmovqb %ymm2, %xmm10
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3,4],ymm6[5],ymm10[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm3, %xmm10
+; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]
+; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm10
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm10, %xmm8
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512DQ-BW-NEXT: vpmovqb %zmm17, %xmm12
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm11
+; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm12
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm1, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm4, %xmm14
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm21
+; AVX512DQ-BW-NEXT: vpsrlq $8, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm3, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm10, %xmm14
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm19
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm15
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpsrlq $16, %ymm2, %ymm15
+; AVX512DQ-BW-NEXT: vpmovqb %ymm15, %xmm15
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm20
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm15
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm19, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpsrlq $24, %ymm2, %ymm15
+; AVX512DQ-BW-NEXT: vpmovqb %ymm15, %xmm15
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm23
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm21
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm14
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT: vpsrlq $32, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm15
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm24
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm22
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm14
+; AVX512DQ-BW-NEXT: vpsrlq $40, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm14
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm25
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, %ymm23
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm2, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm14
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm14
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT: vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm17, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm4, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm8, %xmm14
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-BW-NEXT: vpsrlq $48, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT: vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm16, %ymm0, %ymm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm14, %xmm10, %xmm14
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm9
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm6, %xmm6
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm12, %xmm9
+; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpsrlq $56, %ymm2, %ymm2
+; AVX512DQ-BW-NEXT: vpmovqb %ymm2, %xmm2
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm7, %xmm7
-; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm9, %xmm9
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm13, %xmm9
-; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm12, %xmm7
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm11, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm9, %xmm19, %xmm9
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm1, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm5, %xmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm4, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm6, %xmm8, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm16, %zmm3
-; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm20, (%rsi)
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm21, (%rdx)
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm22, (%rcx)
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm23, (%r8)
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm24, (%r9)
-; AVX512DQ-BW-NEXT: vmovdqa64 %ymm25, (%r11)
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%r10)
-; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rax)
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm3, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm0
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm17, %zmm2
+; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm18, (%rsi)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm19, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm20, (%rcx)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm21, (%r8)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm22, (%r9)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm23, (%r11)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm8, (%r10)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%rax)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -13041,46 +12873,45 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
;
; AVX512-LABEL: load_i8_stride8_vf64:
; AVX512: # %bb.0:
-; AVX512-NEXT: subq $520, %rsp # imm = 0x208
-; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm17
+; AVX512-NEXT: subq $440, %rsp # imm = 0x1B8
+; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm30
; AVX512-NEXT: vmovdqa64 384(%rdi), %zmm0
; AVX512-NEXT: vpmovqb %zmm0, %xmm2
-; AVX512-NEXT: vmovdqa 496(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa 496(%rdi), %xmm6
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm1
-; AVX512-NEXT: vmovdqa 480(%rdi), %xmm8
-; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm3
+; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm1
+; AVX512-NEXT: vmovdqa 480(%rdi), %xmm9
+; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm3
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa 464(%rdi), %xmm13
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm4
-; AVX512-NEXT: vmovdqa 448(%rdi), %xmm15
-; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm5
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm4
+; AVX512-NEXT: vmovdqa 448(%rdi), %xmm8
+; AVX512-NEXT: vpshufb %xmm1, %xmm8, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm4
-; AVX512-NEXT: vpmovqb %ymm4, %xmm4
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm18
+; AVX512-NEXT: vpmovqb %ymm18, %xmm4
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT: vmovdqa 368(%rdi), %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm4, %xmm18
-; AVX512-NEXT: vmovdqa 352(%rdi), %xmm14
-; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm4
+; AVX512-NEXT: vmovdqa 368(%rdi), %xmm10
+; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512-NEXT: vmovdqa 352(%rdi), %xmm12
+; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm4
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT: vmovdqa 336(%rdi), %xmm4
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vmovdqa 320(%rdi), %xmm6
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm5
-; AVX512-NEXT: vmovdqa64 %xmm6, %xmm19
+; AVX512-NEXT: vmovdqa 336(%rdi), %xmm5
+; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
+; AVX512-NEXT: vmovdqa64 %xmm5, %xmm21
+; AVX512-NEXT: vmovdqa 320(%rdi), %xmm7
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm7, %xmm16
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
-; AVX512-NEXT: vpmovqb %zmm17, %xmm4
+; AVX512-NEXT: vpmovqb %zmm30, %xmm4
; AVX512-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
; AVX512-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
; AVX512-NEXT: movb $-64, %al
@@ -13088,18 +12919,15 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
; AVX512-NEXT: vmovdqa 240(%rdi), %xmm4
; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm2
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa64 %xmm4, %xmm25
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm4
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512-NEXT: vmovdqa 224(%rdi), %xmm11
+; AVX512-NEXT: vpshufb %xmm0, %xmm11, %xmm4
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
; AVX512-NEXT: vmovdqa 208(%rdi), %xmm5
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm29
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm10
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm5
-; AVX512-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm5, %xmm22
+; AVX512-NEXT: vmovdqa 192(%rdi), %xmm14
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm5
; AVX512-NEXT: vpmovqb %zmm5, %xmm5
@@ -13107,8 +12935,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm4
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm19
+; AVX512-NEXT: vpmovqb %ymm19, %xmm5
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
@@ -13119,655 +12947,597 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT: vmovdqa 80(%rdi), %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm30
+; AVX512-NEXT: vmovdqa 80(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm5
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm28
+; AVX512-NEXT: vmovdqa64 %xmm5, %xmm27
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512-NEXT: vpmovqb %zmm16, %xmm1
+; AVX512-NEXT: vmovdqa64 (%rdi), %zmm20
+; AVX512-NEXT: vpmovqb %zmm20, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512-NEXT: vpshufb %xmm9, %xmm7, %xmm0
-; AVX512-NEXT: vmovdqa64 %xmm7, %xmm24
-; AVX512-NEXT: vpshufb %xmm9, %xmm8, %xmm2
-; AVX512-NEXT: vmovdqa64 %xmm8, %xmm21
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512-NEXT: vpshufb %xmm5, %xmm6, %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm6, %xmm23
+; AVX512-NEXT: vpshufb %xmm5, %xmm9, %xmm2
+; AVX512-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm9, %xmm31
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512-NEXT: vpshufb %xmm8, %xmm11, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm11, %xmm27
-; AVX512-NEXT: vpshufb %xmm8, %xmm15, %xmm4
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm3
+; AVX512-NEXT: vmovdqa64 %xmm13, %xmm26
+; AVX512-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512-NEXT: vmovdqa %xmm8, %xmm6
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovdqa 416(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa 432(%rdi), %xmm11
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm2
-; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm5
-; AVX512-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512-NEXT: vmovdqa 384(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa 400(%rdi), %xmm12
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm6
-; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm7
-; AVX512-NEXT: vmovdqa64 %xmm0, %xmm31
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm0
-; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm5
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm23
-; AVX512-NEXT: vpshufb %xmm9, %xmm14, %xmm6
-; AVX512-NEXT: vmovdqa64 %xmm14, %xmm26
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512-NEXT: vmovdqa64 %xmm19, %xmm14
-; AVX512-NEXT: vpshufb %xmm8, %xmm14, %xmm7
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512-NEXT: vpsrlq $8, %zmm17, %zmm6
-; AVX512-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm19
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm0
-; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm4
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm9, %xmm0, %xmm5
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT: vmovdqa64 %xmm29, %xmm0
-; AVX512-NEXT: vpshufb %xmm8, %xmm0, %xmm5
-; AVX512-NEXT: vpshufb %xmm8, %xmm10, %xmm6
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm25
-; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm0, %xmm20
; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-NEXT: vmovdqa 432(%rdi), %xmm7
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm4
+; AVX512-NEXT: vmovdqa64 %xmm7, %xmm29
+; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpsrlq $8, %ymm18, %ymm9
+; AVX512-NEXT: vpmovqb %ymm9, %xmm9
+; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT: vpshufb %xmm5, %xmm10, %xmm4
+; AVX512-NEXT: vmovdqa64 %xmm10, %xmm24
+; AVX512-NEXT: vpshufb %xmm5, %xmm12, %xmm9
+; AVX512-NEXT: vmovdqa %xmm12, %xmm13
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT: vmovdqa64 %xmm21, %xmm12
+; AVX512-NEXT: vpshufb %xmm1, %xmm12, %xmm9
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm0
+; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm28
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm9[0,1,2],xmm4[3]
+; AVX512-NEXT: vpsrlq $8, %zmm30, %zmm9
+; AVX512-NEXT: vpmovqb %zmm9, %xmm9
+; AVX512-NEXT: vpblendd {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm7
+; AVX512-NEXT: vpshufb %xmm5, %xmm7, %xmm3
+; AVX512-NEXT: vpshufb %xmm5, %xmm11, %xmm4
+; AVX512-NEXT: vmovdqa %xmm11, %xmm15
+; AVX512-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm11
+; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm4
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm14, %xmm22
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512-NEXT: vmovdqa 160(%rdi), %xmm14
+; AVX512-NEXT: vmovdqa 176(%rdi), %xmm8
+; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm0
+; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm9, %xmm1, %xmm1
+; AVX512-NEXT: vpsrlq $8, %ymm19, %ymm2
+; AVX512-NEXT: vpmovqb %ymm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm9, %xmm2, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm7
-; AVX512-NEXT: vpshufb %xmm8, %xmm7, %xmm2
-; AVX512-NEXT: vmovdqa64 %xmm28, %xmm3
-; AVX512-NEXT: vpshufb %xmm8, %xmm3, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT: vpsrlq $8, %zmm16, %zmm2
+; AVX512-NEXT: vpshufb %xmm5, %xmm2, %xmm2
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm5, %xmm3, %xmm5
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm27, %xmm3
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
+; AVX512-NEXT: vpsrlq $8, %zmm20, %zmm2
; AVX512-NEXT: vpmovqb %zmm2, %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm19, %zmm0
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm13
-; AVX512-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm4, %xmm13, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm10
-; AVX512-NEXT: vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm4, %xmm10, %xmm2
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
+; AVX512-NEXT: vmovdqa64 %xmm23, %xmm4
+; AVX512-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm31, %xmm2
+; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm0
-; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm8
-; AVX512-NEXT: vmovdqa64 %xmm15, %xmm18
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; AVX512-NEXT: vmovdqa64 %xmm26, %xmm3
+; AVX512-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
+; AVX512-NEXT: vmovdqa64 %xmm6, %xmm16
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm2
-; AVX512-NEXT: vmovdqa64 %xmm11, %xmm24
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm11
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm0
-; AVX512-NEXT: vmovdqa64 %xmm12, %xmm30
-; AVX512-NEXT: vmovdqa64 %xmm31, %xmm5
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm23, %xmm12
-; AVX512-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm4, %xmm12, %xmm8
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT: vpshufb %xmm4, %xmm5, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm15
-; AVX512-NEXT: vmovdqa64 %xmm14, %xmm21
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $16, %zmm17, %zmm9
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm0
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm4, %xmm14, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
; AVX512-NEXT: vmovdqa64 %xmm29, %xmm6
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm9
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm9
-; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm15
-; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm2, %xmm15, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT: vpsrlq $16, %ymm18, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa64 %xmm24, %xmm6
+; AVX512-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
+; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm13, %xmm31
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm12, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm21, %xmm23
+; AVX512-NEXT: vmovdqa64 %xmm28, %xmm12
+; AVX512-NEXT: vpshufb %xmm1, %xmm12, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT: vmovdqa64 %zmm30, %zmm21
+; AVX512-NEXT: vpsrlq $16, %zmm30, %zmm10
+; AVX512-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm17
+; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm11, %xmm24
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm13
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm8, %xmm22
+; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm2
+; AVX512-NEXT: vmovdqa64 %xmm14, %xmm25
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm7, %xmm29
-; AVX512-NEXT: vmovdqa64 %xmm28, %xmm7
-; AVX512-NEXT: vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
+; AVX512-NEXT: vpsrlq $16, %ymm19, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT: vmovdqa (%rsp), %xmm11 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm27, %xmm7
; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512-NEXT: vpsrlq $16, %zmm16, %zmm2
-; AVX512-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX512-NEXT: vpsrlq $16, %zmm20, %zmm1
+; AVX512-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm2
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm10
-; AVX512-NEXT: vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm4
-; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm8
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm28
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm9
-; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm31, %xmm11
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vmovdqa64 %xmm29, %xmm8
+; AVX512-NEXT: vmovdqa64 %xmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm9
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm8
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm13
-; AVX512-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $24, %zmm17, %zmm9
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
+; AVX512-NEXT: vpsrlq $24, %ymm18, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
+; AVX512-NEXT: vmovdqa64 %xmm31, %xmm3
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vmovdqa64 %xmm23, %xmm3
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm28, %xmm3
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT: vpsrlq $24, %zmm30, %zmm10
+; AVX512-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vmovdqa64 %xmm17, %xmm4
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm5
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm14, %xmm22
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm6, %xmm26
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm14
-; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT: vmovdqa64 %xmm24, %xmm4
+; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm10
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm12
+; AVX512-NEXT: vmovdqa64 %xmm13, %xmm26
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm4
+; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm0
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT: vpsrlq $24, %ymm19, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm14, %xmm28
+; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm15, %xmm29
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm11, %xmm30
+; AVX512-NEXT: vmovdqa64 %xmm27, %xmm4
+; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $24, %zmm16, %zmm1
+; AVX512-NEXT: vpsrlq $24, %zmm20, %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm10
-; AVX512-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm15
+; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm11
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm28, %xmm7
-; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm8
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm7
-; AVX512-NEXT: vpshufb %xmm3, %xmm7, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm31, %xmm12
-; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm9
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT: vpsrlq $32, %ymm18, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm8
-; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm9
+; AVX512-NEXT: vmovdqa64 %xmm31, %xmm8
+; AVX512-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vmovdqa64 %xmm23, %xmm13
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm10
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT: vmovdqa64 %xmm3, %xmm31
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT: vmovdqa64 %zmm21, %zmm23
+; AVX512-NEXT: vpsrlq $32, %zmm21, %zmm10
+; AVX512-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vmovdqa64 %xmm17, %xmm3
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm5
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm9
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $32, %zmm17, %zmm9
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm5
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vpshufb %xmm2, %xmm14, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm20
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT: vmovdqa64 %xmm24, %xmm14
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm26, %xmm7
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm11
+; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm7
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm3
-; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm0
-; AVX512-NEXT: vmovdqa64 %xmm6, %xmm25
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vpsrlq $32, %ymm19, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa64 %xmm28, %xmm5
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm29, %xmm10
+; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT: vmovdqa64 %xmm30, %xmm5
+; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm5
; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $32, %zmm16, %zmm1
-; AVX512-NEXT: vmovdqa64 %zmm16, %zmm18
+; AVX512-NEXT: vpsrlq $32, %zmm20, %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
+; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm2
+; AVX512-NEXT: vmovdqa64 %xmm6, %xmm21
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm5
+; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm9
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm22
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm27
-; AVX512-NEXT: vmovdqa64 %xmm28, %xmm5
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm8
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm10
-; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm9
-; AVX512-NEXT: vpshufb %xmm3, %xmm12, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm8
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm11
-; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $40, %zmm17, %zmm9
-; AVX512-NEXT: vmovdqa64 %zmm17, %zmm23
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm4
-; AVX512-NEXT: vmovdqa64 %xmm13, %xmm30
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm6
-; AVX512-NEXT: vpshufb %xmm0, %xmm6, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT: vmovdqa64 %xmm26, %xmm13
-; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm9
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm2, %xmm4, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm9
-; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm9
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; AVX512-NEXT: vpsrlq $40, %ymm18, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm9
+; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm13, %xmm24
+; AVX512-NEXT: vmovdqa64 %xmm31, %xmm4
+; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT: vpsrlq $40, %zmm23, %zmm10
+; AVX512-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm17, %xmm25
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm3, %xmm15, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm15, %xmm28
-; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm14, %xmm16
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm14, %xmm17
+; AVX512-NEXT: vmovdqa64 %xmm26, %xmm3
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm11, %xmm16
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT: vmovdqa64 %xmm7, %xmm26
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm7, %xmm24
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vmovdqa64 %xmm29, %xmm7
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT: vpsrlq $40, %ymm19, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm29, %xmm8
+; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT: vmovdqa64 %xmm30, %xmm6
+; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm27, %xmm7
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $40, %zmm18, %zmm1
-; AVX512-NEXT: vmovdqa64 %zmm18, %zmm26
+; AVX512-NEXT: vpsrlq $40, %zmm20, %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm1
-; AVX512-NEXT: vmovdqa64 %xmm2, %xmm29
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm11, %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm21, %xmm2
; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm9
+; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm8
-; AVX512-NEXT: vmovdqa64 %xmm5, %xmm21
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm10, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm10, %xmm18
-; AVX512-NEXT: vmovdqa64 %xmm31, %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm5, %xmm15
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm8
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm9
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm9
-; AVX512-NEXT: vpshufb %xmm1, %xmm11, %xmm15
-; AVX512-NEXT: vmovdqa64 %xmm11, %xmm22
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $48, %zmm17, %zmm9
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm9, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm13, %xmm19
-; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT: vmovdqa64 %xmm6, %xmm17
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT: vpshufb %xmm2, %xmm10, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm6
-; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm9
+; AVX512-NEXT: vpsrlq $48, %ymm18, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm9
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm10
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT: vmovdqa64 %xmm24, %xmm10
+; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm31, %xmm14
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm12
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT: vpsrlq $48, %zmm23, %zmm10
+; AVX512-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm5
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm15, %xmm28
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT: vmovdqa64 %xmm17, %xmm15
+; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm10
+; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT: vmovdqa64 %xmm3, %xmm17
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm10
+; AVX512-NEXT: vmovdqa64 %xmm26, %xmm12
; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm2
-; AVX512-NEXT: vmovdqa64 %xmm12, %xmm20
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT: vmovdqa64 %xmm28, %xmm14
-; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm16, %xmm13
-; AVX512-NEXT: vpshufb %xmm3, %xmm13, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vmovdqa64 %xmm24, %xmm15
-; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm25, %xmm12
-; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm0
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512-NEXT: vmovdqa (%rsp), %xmm7 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT: vpsrlq $48, %ymm19, %ymm10
+; AVX512-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm5
+; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm0
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm27, %xmm10
+; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm1
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $48, %zmm26, %zmm1
+; AVX512-NEXT: vpsrlq $48, %zmm20, %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm24
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm26
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512-NEXT: vmovdqa64 %xmm29, %xmm1
-; AVX512-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm11, %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm21, %xmm2
; AVX512-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm22, %xmm6
+; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm27, %xmm3
-; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa64 %xmm21, %xmm8
-; AVX512-NEXT: vpshufb %xmm2, %xmm8, %xmm8
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT: vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vmovdqa64 %xmm18, %xmm9
-; AVX512-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm31, %xmm11
-; AVX512-NEXT: vpshufb %xmm3, %xmm11, %xmm11
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm8
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm22, %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm11
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT: vpsrlq $56, %zmm23, %zmm9
-; AVX512-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT: vmovdqa64 %xmm19, %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT: vmovdqa64 %xmm17, %xmm5
-; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm11
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm6
-; AVX512-NEXT: vmovdqa64 %xmm20, %xmm5
-; AVX512-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm5
-; AVX512-NEXT: vpshufb %xmm3, %xmm13, %xmm3
-; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm2, %xmm7, %xmm7
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT: vpsrlq $56, %ymm18, %ymm7
+; AVX512-NEXT: vpmovqb %ymm7, %xmm7
+; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512-NEXT: vmovdqa64 %xmm24, %xmm7
+; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm7
+; AVX512-NEXT: vpshufb %xmm1, %xmm14, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
+; AVX512-NEXT: vpsrlq $56, %zmm23, %zmm7
+; AVX512-NEXT: vpmovqb %zmm7, %xmm7
+; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
+; AVX512-NEXT: vinserti32x4 $2, %xmm6, %zmm0, %zmm6
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm6 {%k1}
+; AVX512-NEXT: vmovdqa64 %xmm25, %xmm5
+; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT: vmovdqa64 %xmm28, %xmm7
+; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm7
+; AVX512-NEXT: vmovdqa64 %xmm17, %xmm9
+; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm9
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm3
+; AVX512-NEXT: vpshufb %xmm2, %xmm12, %xmm2
+; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT: vpsrlq $56, %ymm19, %ymm3
+; AVX512-NEXT: vpmovqb %ymm3, %xmm3
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %xmm0, %xmm15, %xmm3
-; AVX512-NEXT: vpshufb %xmm0, %xmm12, %xmm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT: vpshufb %xmm0, %xmm4, %xmm3
+; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm0
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT: vmovdqa64 %xmm30, %xmm3
; AVX512-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm1
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT: vpsrlq $56, %zmm26, %zmm1
+; AVX512-NEXT: vpsrlq $56, %zmm20, %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm6, %zmm0
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovaps %zmm1, (%rsi)
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -13782,10 +13552,10 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovaps %zmm1, (%rax)
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT: vmovdqa64 %zmm24, (%rax)
+; AVX512-NEXT: vmovdqa64 %zmm26, (%rax)
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: vmovdqa64 %zmm0, (%rax)
-; AVX512-NEXT: addq $520, %rsp # imm = 0x208
+; AVX512-NEXT: addq $440, %rsp # imm = 0x1B8
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -14218,46 +13988,45 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
;
; AVX512DQ-LABEL: load_i8_stride8_vf64:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: subq $520, %rsp # imm = 0x208
-; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm17
+; AVX512DQ-NEXT: subq $440, %rsp # imm = 0x1B8
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm30
; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %zmm0
; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm2
-; AVX512DQ-NEXT: vmovdqa 496(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa 496(%rdi), %xmm6
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm1
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm1
+; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm3
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm13
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm4
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm15
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm4
+; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm8, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm4
-; AVX512DQ-NEXT: vpmovqb %ymm4, %xmm4
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm18
+; AVX512DQ-NEXT: vpmovqb %ymm18, %xmm4
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm18
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm4
+; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm12
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm4
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm5
-; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm19
+; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
+; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm21
+; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm16
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
-; AVX512DQ-NEXT: vpmovqb %zmm17, %xmm4
+; AVX512DQ-NEXT: vpmovqb %zmm30, %xmm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
; AVX512DQ-NEXT: movb $-64, %al
@@ -14265,18 +14034,15 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm4
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm25
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm11
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm11, %xmm4
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm5
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm29
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm10
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm5
-; AVX512DQ-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm22
+; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm5
; AVX512DQ-NEXT: vpmovqb %zmm5, %xmm5
@@ -14284,8 +14050,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512DQ-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm19
+; AVX512DQ-NEXT: vpmovqb %ymm19, %xmm5
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
@@ -14296,655 +14062,597 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm30
+; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm5
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm28
+; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm27
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-NEXT: vpmovqb %zmm16, %xmm1
+; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm20
+; AVX512DQ-NEXT: vpmovqb %zmm20, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm7, %xmm0
-; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm24
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm8, %xmm2
-; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm21
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm5 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm6, %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm23
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm9, %xmm2
+; AVX512DQ-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm31
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm11, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm27
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm15, %xmm4
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm3
+; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm26
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm8, %xmm4
+; AVX512DQ-NEXT: vmovdqa %xmm8, %xmm6
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovdqa 416(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa 432(%rdi), %xmm11
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm5
-; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm22
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa 400(%rdi), %xmm12
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm7
-; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm31
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm5
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm23
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm14, %xmm6
-; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm26
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm0, %xmm6
-; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm14, %xmm7
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512DQ-NEXT: vpsrlq $8, %zmm17, %zmm6
-; AVX512DQ-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm19
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm4
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm0, %xmm5
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm0, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm10, %xmm6
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm25
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm20
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-NEXT: vmovdqa 432(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm4
+; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm29
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpsrlq $8, %ymm18, %ymm9
+; AVX512DQ-NEXT: vpmovqb %ymm9, %xmm9
+; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm10, %xmm4
+; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm24
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm12, %xmm9
+; AVX512DQ-NEXT: vmovdqa %xmm12, %xmm13
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm12
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm12, %xmm9
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm0
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm28
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm9[0,1,2],xmm4[3]
+; AVX512DQ-NEXT: vpsrlq $8, %zmm30, %zmm9
+; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm7, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm11, %xmm4
+; AVX512DQ-NEXT: vmovdqa %xmm11, %xmm15
+; AVX512DQ-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm11
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm22
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm14
+; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm0
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpsrlq $8, %ymm19, %ymm2
+; AVX512DQ-NEXT: vpmovqb %ymm2, %xmm2
+; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm9, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm7, %xmm2
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm8, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT: vpsrlq $8, %zmm16, %zmm2
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm2, %xmm2
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm5, %xmm3, %xmm5
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
+; AVX512DQ-NEXT: vpsrlq $8, %zmm20, %zmm2
; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm19, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm13
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm13, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm10
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm10, %xmm2
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
+; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm4
+; AVX512DQ-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm2
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm0
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm8
-; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm18
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm3
+; AVX512DQ-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
+; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm16
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm2
-; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm24
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm0
-; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm30
-; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm12
-; AVX512DQ-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm12, %xmm8
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm5, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm15
-; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm21
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm17, %zmm9
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm0, %xmm0
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm14, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm9
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm15
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm15, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT: vpsrlq $16, %ymm18, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm6
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm31
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm12, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm23
+; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm12
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm12, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa64 %zmm30, %zmm21
+; AVX512DQ-NEXT: vpsrlq $16, %zmm30, %zmm10
+; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm17
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm24
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm22
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm2
+; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm25
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm4, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm29
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm7
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT: vpsrlq $16, %ymm19, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm11 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm7
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512DQ-NEXT: vpsrlq $16, %zmm16, %zmm2
-; AVX512DQ-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX512DQ-NEXT: vpsrlq $16, %zmm20, %zmm1
+; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm2
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm1
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm10
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm8
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm28
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm8
+; AVX512DQ-NEXT: vmovdqa64 %xmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm9
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm8
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm13
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $24, %zmm17, %zmm9
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
+; AVX512DQ-NEXT: vpsrlq $24, %ymm18, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT: vpsrlq $24, %zmm30, %zmm10
+; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm5
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm22
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm26
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm12
+; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm26
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm0
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT: vpsrlq $24, %ymm19, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm28
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm29
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm30
+; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $24, %zmm16, %zmm1
+; AVX512DQ-NEXT: vpsrlq $24, %zmm20, %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm10
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm8
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm7, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm12
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vpsrlq $32, %ymm18, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm9
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm8
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm13
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm31
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa64 %zmm21, %zmm23
+; AVX512DQ-NEXT: vpsrlq $32, %zmm21, %zmm10
+; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm5
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm17, %zmm9
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm14, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm20
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm11
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm0
-; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm25
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vpsrlq $32, %ymm19, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm5
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $32, %zmm16, %zmm1
-; AVX512DQ-NEXT: vmovdqa64 %zmm16, %zmm18
+; AVX512DQ-NEXT: vpsrlq $32, %zmm20, %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm2
+; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm21
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm9
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm22
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm27
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm8
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm10
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm12, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm8
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $40, %zmm17, %zmm9
-; AVX512DQ-NEXT: vmovdqa64 %zmm17, %zmm23
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm4
-; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm30
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm6, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm13
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm9
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm4, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm9
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; AVX512DQ-NEXT: vpsrlq $40, %ymm18, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm24
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT: vpsrlq $40, %zmm23, %zmm10
+; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm25
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm15, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm28
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm16
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm17
+; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm16
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm26
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm24
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm7
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT: vpsrlq $40, %ymm19, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm4
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm8
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm6
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $40, %zmm18, %zmm1
-; AVX512DQ-NEXT: vmovdqa64 %zmm18, %zmm26
+; AVX512DQ-NEXT: vpsrlq $40, %zmm20, %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm1
-; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm29
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm11, %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm2
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm8
-; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm21
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm10, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm18
-; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm5, %xmm15
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm8
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm9
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm11, %xmm15
-; AVX512DQ-NEXT: vmovdqa64 %xmm11, %xmm22
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm17, %zmm9
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm9, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm19
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm17
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm10, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm6
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm9
+; AVX512DQ-NEXT: vpsrlq $48, %ymm18, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm9
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm10
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm14
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm12
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT: vpsrlq $48, %zmm23, %zmm10
+; AVX512DQ-NEXT: vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm28
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm15
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm17
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm10
+; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm12
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm2
-; AVX512DQ-NEXT: vmovdqa64 %xmm12, %xmm20
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm14
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm13
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm13, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm15
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm12
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm0
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm7, %xmm16
-; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT: vpsrlq $48, %ymm19, %ymm10
+; AVX512DQ-NEXT: vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm0
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm10
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm1
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $48, %zmm26, %zmm1
+; AVX512DQ-NEXT: vpsrlq $48, %zmm20, %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm24
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm9, %zmm26
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512DQ-NEXT: vmovdqa64 %xmm29, %xmm1
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm11, %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm2
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm2, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm6
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm8
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm8, %xmm8
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT: vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vmovdqa64 %xmm18, %xmm9
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm31, %xmm11
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm11, %xmm11
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm8
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm11
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT: vpsrlq $56, %zmm23, %zmm9
-; AVX512DQ-NEXT: vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm11
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm6
-; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm5
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm13, %xmm3
-; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm7, %xmm7
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpsrlq $56, %ymm18, %ymm7
+; AVX512DQ-NEXT: vpmovqb %ymm7, %xmm7
+; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm13, %xmm6
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm14, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
+; AVX512DQ-NEXT: vpsrlq $56, %zmm23, %zmm7
+; AVX512DQ-NEXT: vpmovqb %zmm7, %xmm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $2, %xmm6, %zmm0, %zmm6
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm6 {%k1}
+; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm5
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT: vmovdqa64 %xmm28, %xmm7
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm7
+; AVX512DQ-NEXT: vmovdqa64 %xmm17, %xmm9
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm9
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm2, %xmm12, %xmm2
+; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpsrlq $56, %ymm19, %ymm3
+; AVX512DQ-NEXT: vpmovqb %ymm3, %xmm3
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm15, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm0, %xmm12, %xmm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm4, %xmm3
+; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm0
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm3
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT: vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm1
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT: vpsrlq $56, %zmm26, %zmm1
+; AVX512DQ-NEXT: vpsrlq $56, %zmm20, %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm6, %zmm0
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovaps %zmm1, (%rsi)
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -14959,10 +14667,10 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovaps %zmm1, (%rax)
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT: vmovdqa64 %zmm24, (%rax)
+; AVX512DQ-NEXT: vmovdqa64 %zmm26, (%rax)
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-NEXT: addq $520, %rsp # imm = 0x208
+; AVX512DQ-NEXT: addq $440, %rsp # imm = 0x1B8
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -15395,678 +15103,621 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
;
; AVX512BW-LABEL: load_i8_stride8_vf64:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: subq $744, %rsp # imm = 0x2E8
-; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm1
+; AVX512BW-NEXT: subq $552, %rsp # imm = 0x228
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm10
+; AVX512BW-NEXT: vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT: vpmovqb %zmm1, %xmm12
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm0
; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512BW-NEXT: vmovdqa64 496(%rdi), %xmm24
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm24, %xmm3
+; AVX512BW-NEXT: vpmovqb %zmm2, %xmm3
+; AVX512BW-NEXT: vmovdqa64 496(%rdi), %xmm27
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm27, %xmm4
; AVX512BW-NEXT: vmovdqa64 480(%rdi), %xmm25
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm25, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm26
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm26, %xmm4
-; AVX512BW-NEXT: vmovdqa64 448(%rdi), %xmm30
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm30, %xmm6
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
-; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm4
-; AVX512BW-NEXT: vpmovqb %ymm4, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm25, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512BW-NEXT: vmovdqa64 368(%rdi), %xmm31
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm31, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa64 352(%rdi), %xmm27
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm27, %xmm6
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512BW-NEXT: vmovdqa64 336(%rdi), %xmm22
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm22, %xmm11
-; AVX512BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa 320(%rdi), %xmm9
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm9, %xmm15
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
-; AVX512BW-NEXT: vpmovqb %zmm1, %xmm11
-; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm15
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
-; AVX512BW-NEXT: vinserti32x4 $2, %xmm10, %zmm0, %zmm20
+; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm31
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm18 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm31, %xmm5
+; AVX512BW-NEXT: vmovdqa64 448(%rdi), %xmm28
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm28, %xmm8
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
+; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm5
+; AVX512BW-NEXT: vmovdqa64 384(%rdi), %ymm16
+; AVX512BW-NEXT: vpmovqb %ymm16, %xmm8
+; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4],ymm5[5],ymm8[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512BW-NEXT: vmovdqa64 368(%rdi), %xmm23
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm23, %xmm8
+; AVX512BW-NEXT: vmovdqa64 352(%rdi), %xmm24
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm24, %xmm9
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512BW-NEXT: vmovdqa64 336(%rdi), %xmm26
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm26, %xmm14
+; AVX512BW-NEXT: vmovdqa64 320(%rdi), %xmm19
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm19, %xmm15
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
+; AVX512BW-NEXT: vpmovqb %zmm0, %xmm14
+; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm9
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
+; AVX512BW-NEXT: vinserti32x4 $2, %xmm13, %zmm0, %zmm21
; AVX512BW-NEXT: movb $-64, %al
; AVX512BW-NEXT: kmovd %eax, %k1
-; AVX512BW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
-; AVX512BW-NEXT: vmovdqa64 240(%rdi), %xmm28
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm28, %xmm7
-; AVX512BW-NEXT: vmovdqa64 224(%rdi), %xmm18
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm18, %xmm10
-; AVX512BW-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT: vmovdqa64 208(%rdi), %xmm17
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm17, %xmm10
-; AVX512BW-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm8
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm8, %xmm16
-; AVX512BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
-; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm10
-; AVX512BW-NEXT: vpmovqb %ymm10, %xmm10
-; AVX512BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
-; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm10
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm10, %xmm16
-; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm14
-; AVX512BW-NEXT: vpshufb %xmm12, %xmm14, %xmm12
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
-; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm2
-; AVX512BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm2, %xmm21
-; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm2, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpmovqb %zmm5, %xmm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm21 {%k1}
+; AVX512BW-NEXT: vmovdqa 240(%rdi), %xmm0
+; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm0, %xmm13
+; AVX512BW-NEXT: vmovdqa 224(%rdi), %xmm0
+; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm0, %xmm15
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm6
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm6, %xmm15
+; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm14
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm14, %xmm20
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
+; AVX512BW-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm15
+; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vpmovqb %ymm0, %xmm20
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm20, %ymm0, %ymm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm13
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm13, %xmm20
+; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm15
+; AVX512BW-NEXT: vpshufb %xmm17, %xmm15, %xmm17
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm17[0],xmm20[0],xmm17[1],xmm20[1],xmm17[2],xmm20[2],xmm17[3],xmm20[3]
+; AVX512BW-NEXT: vmovdqa 80(%rdi), %xmm12
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm12, %xmm22
+; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm7
+; AVX512BW-NEXT: vpshufb %xmm18, %xmm7, %xmm18
+; AVX512BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm18[0],xmm22[0],xmm18[1],xmm22[1],xmm18[2],xmm22[2],xmm18[3],xmm22[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpmovqb %zmm10, %xmm2
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm21, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm6
-; AVX512BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %xmm7
-; AVX512BW-NEXT: vmovdqa64 400(%rdi), %xmm21
-; AVX512BW-NEXT: vmovdqa64 416(%rdi), %xmm23
-; AVX512BW-NEXT: vmovdqa64 432(%rdi), %xmm29
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm19
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm1
-; AVX512BW-NEXT: vmovdqa64 %xmm25, %xmm11
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vmovdqa64 %xmm26, %xmm12
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm26, %xmm24
-; AVX512BW-NEXT: vmovdqa64 %xmm30, %xmm16
-; AVX512BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm30, %xmm25
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm26, %xmm29, %xmm24
-; AVX512BW-NEXT: vpshufb %xmm26, %xmm23, %xmm25
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm21, %xmm24
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm7, %xmm25
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm31, %xmm2
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm27, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm22, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm9, %xmm24
-; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm31
-; AVX512BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
-; AVX512BW-NEXT: vpsrlq $8, %zmm15, %zmm3
-; AVX512BW-NEXT: vmovdqa64 %zmm15, %zmm22
-; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
-; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT: vmovdqa64 %xmm28, %xmm9
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm28, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm18, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm17, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm8, %xmm24
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm0
-; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm20
+; AVX512BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm22
+; AVX512BW-NEXT: vmovdqa64 416(%rdi), %xmm29
+; AVX512BW-NEXT: vmovdqa 432(%rdi), %xmm8
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm27, %xmm1
+; AVX512BW-NEXT: vmovdqa64 %xmm27, %xmm10
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm31, %xmm27
+; AVX512BW-NEXT: vmovdqa64 %xmm28, %xmm11
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm28, %xmm28
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm3
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpshufb %xmm26, %xmm0, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm26, %xmm6, %xmm25
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm6
-; AVX512BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm0
-; AVX512BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm6, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm10, %xmm1
-; AVX512BW-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm14, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm15, %xmm3
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm28, %xmm4
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT: vpshufb %xmm30, %xmm8, %xmm27
+; AVX512BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm30, %xmm29, %xmm28
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm3
+; AVX512BW-NEXT: vpsrlq $8, %ymm16, %ymm27
+; AVX512BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm4
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm24, %xmm4
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT: vpsrlq $8, %zmm5, %zmm3
-; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm26, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm19, %xmm18
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm19, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
+; AVX512BW-NEXT: vpsrlq $8, %zmm9, %zmm4
+; AVX512BW-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
+; AVX512BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
+; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm17, %xmm4
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm14, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT: vpshufb %xmm30, %xmm22, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm22, %xmm19
+; AVX512BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm30, %xmm20, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm22 # 32-byte Reload
+; AVX512BW-NEXT: vpsrlq $8, %ymm22, %ymm27
+; AVX512BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm13, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm12, %xmm20
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT: vpsrlq $8, %zmm12, %zmm2
+; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm19, %xmm2
-; AVX512BW-NEXT: vmovdqa64 %xmm19, %xmm20
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm11, %xmm3
+; AVX512BW-NEXT: vmovdqa %xmm10, %xmm7
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm10, %xmm2
+; AVX512BW-NEXT: vmovdqa64 %xmm25, %xmm28
+; AVX512BW-NEXT: vmovdqa64 %xmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm25, %xmm3
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm12, %xmm25
-; AVX512BW-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT: vmovdqa64 %xmm31, %xmm30
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm11, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm31
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vmovdqa64 %xmm29, %xmm24
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm29, %xmm13
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm23, %xmm30
-; AVX512BW-NEXT: vmovdqa64 %xmm23, %xmm16
-; AVX512BW-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT: vmovdqa64 %xmm21, %xmm18
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm21, %xmm0
-; AVX512BW-NEXT: vmovdqa64 %xmm7, %xmm17
-; AVX512BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm7, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm8, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm29, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vmovdqa64 %ymm16, %ymm10
+; AVX512BW-NEXT: vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vpsrlq $16, %ymm16, %ymm27
+; AVX512BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm0
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm12, %xmm2
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm27, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm13
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512BW-NEXT: vmovdqa64 %zmm22, %zmm6
-; AVX512BW-NEXT: vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vpsrlq $16, %zmm22, %zmm13
-; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
+; AVX512BW-NEXT: vmovdqa64 %xmm23, %xmm25
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm23, %xmm2
+; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm16
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT: vmovdqa64 %xmm26, %xmm8
+; AVX512BW-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm18, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT: vpsrlq $16, %zmm9, %zmm5
+; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm9, %xmm0
-; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm21, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm13
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm21, %xmm0
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm17, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm6, %xmm5
+; AVX512BW-NEXT: vmovdqa %xmm6, %xmm11
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm14, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm13
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm7, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm31, %xmm13
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm30, %xmm23, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm19, %xmm5
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm24 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm4
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512BW-NEXT: vpsrlq $16, %ymm22, %ymm5
+; AVX512BW-NEXT: vmovdqa64 %ymm22, %ymm23
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm14, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm13, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm13, %xmm19
+; AVX512BW-NEXT: vpshufb %xmm1, %xmm15, %xmm1
+; AVX512BW-NEXT: vmovdqa64 %xmm15, %xmm22
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm28, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm3
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT: vpsrlq $16, %zmm5, %zmm3
-; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm28
+; AVX512BW-NEXT: vpsrlq $16, %zmm12, %zmm3
+; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm6
; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm7, %xmm2
+; AVX512BW-NEXT: vmovdqa %xmm7, %xmm15
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm28, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm30, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT: vmovdqa64 %xmm31, %xmm28
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm13, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm29, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpsrlq $24, %ymm10, %ymm27
+; AVX512BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm1
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5],ymm1[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm18, (%rsp) # 16-byte Spill
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm18, %xmm27
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT: vpsrlq $24, %zmm9, %zmm5
+; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm11, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm14, %xmm27
+; AVX512BW-NEXT: vmovdqa64 %xmm14, %xmm17
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm7, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm31
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm4
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT: vmovdqa64 %ymm23, %ymm12
+; AVX512BW-NEXT: vpsrlq $24, %ymm23, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vmovdqa64 %xmm19, %xmm10
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm19, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm26, %xmm24
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $24, %zmm6, %zmm3
+; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm25, %xmm3
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm30, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm28, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm28, %xmm23
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm24, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm17, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm13, %xmm4
+; AVX512BW-NEXT: vmovdqa64 %xmm13, %xmm19
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Reload
+; AVX512BW-NEXT: vpsrlq $32, %ymm30, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vmovdqa %xmm12, %xmm10
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm12, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm13
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
-; AVX512BW-NEXT: vpsrlq $24, %zmm6, %zmm13
-; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm18, %xmm28
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512BW-NEXT: vpsrlq $32, %zmm9, %zmm5
+; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm26, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm13
-; AVX512BW-NEXT: vmovdqa64 %xmm22, %xmm21
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm30
-; AVX512BW-NEXT: vmovdqa64 %xmm29, %xmm22
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm11, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm18
+; AVX512BW-NEXT: vmovdqa64 %xmm17, %xmm26
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm17, %xmm28
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm9, %xmm13
-; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm25
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512BW-NEXT: vmovdqa %xmm7, %xmm9
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm23, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm7, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm7, %xmm28
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm3
+; AVX512BW-NEXT: vmovdqa64 %xmm31, %xmm11
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
+; AVX512BW-NEXT: vpsrlq $32, %ymm12, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm5, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm0
-; AVX512BW-NEXT: vmovdqa64 %xmm14, %xmm23
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm15, %xmm3
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vmovdqa64 %zmm28, %zmm17
-; AVX512BW-NEXT: vpsrlq $24, %zmm28, %zmm2
+; AVX512BW-NEXT: vpsrlq $32, %zmm6, %zmm2
+; AVX512BW-NEXT: vmovdqa64 %zmm6, %zmm7
; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
-; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm28
-; AVX512BW-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm24, %xmm13
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm26, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
-; AVX512BW-NEXT: vmovdqa64 %xmm18, %xmm24
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm18, %xmm2
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vmovdqa %xmm10, %xmm6
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm2
-; AVX512BW-NEXT: vmovdqa64 %xmm27, %xmm7
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm10, %xmm13
-; AVX512BW-NEXT: vmovdqa64 %xmm8, %xmm27
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT: vpsrlq $32, %zmm10, %zmm13
-; AVX512BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
-; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm21, %xmm13
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm19, %xmm8, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm5, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vmovdqa64 %zmm17, %zmm20
-; AVX512BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vpsrlq $32, %zmm17, %zmm3
-; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm28, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT: vmovdqa %xmm8, %xmm13
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm12, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm16, %xmm4
+; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm17, %xmm3
+; AVX512BW-NEXT: vmovdqa64 %xmm23, %xmm6
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm23, %xmm4
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm24, %xmm19
-; AVX512BW-NEXT: vmovdqa64 %xmm24, %xmm26
-; AVX512BW-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512BW-NEXT: vmovdqa64 %xmm19, %xmm31
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm19, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm5
+; AVX512BW-NEXT: vpsrlq $40, %ymm30, %ymm5
+; AVX512BW-NEXT: vmovdqa64 %ymm30, %ymm19
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm6, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm6, %xmm24
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm5, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT: vmovdqa (%rsp), %xmm8 # 16-byte Reload
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
-; AVX512BW-NEXT: vpsrlq $40, %zmm10, %zmm5
+; AVX512BW-NEXT: vpsrlq $40, %zmm9, %zmm5
; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm5
-; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm17
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT: vmovdqa64 %xmm18, %xmm23
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm18, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm26, %xmm30
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm25, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm9, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm28, %xmm5
+; AVX512BW-NEXT: vmovdqa64 %xmm11, %xmm18
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm11, %xmm3
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm8, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT: vpsrlq $40, %ymm12, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vpsrlq $40, %zmm20, %zmm2
+; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm11
+; AVX512BW-NEXT: vpsrlq $40, %zmm7, %zmm2
; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm28, %xmm1
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm16, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm13, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm17, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm6, %xmm5
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm5
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm26, %xmm19
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm31, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm29, %xmm30
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm24, %xmm3
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm30, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
-; AVX512BW-NEXT: vpsrlq $48, %zmm10, %zmm5
+; AVX512BW-NEXT: vpsrlq $48, %ymm19, %ymm30
+; AVX512BW-NEXT: vpmovqb %ymm30, %xmm30
+; AVX512BW-NEXT: vinserti32x4 $1, %xmm30, %ymm0, %ymm2
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm27, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT: vpsrlq $48, %zmm9, %zmm5
; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
-; AVX512BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
-; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm23, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm30
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm9, %xmm18
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm28, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
-; AVX512BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm13, %xmm20, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT: vpsrlq $48, %ymm12, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm11, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512BW-NEXT: vmovdqa64 %xmm6, %xmm26
-; AVX512BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT: vpsrlq $48, %zmm19, %zmm2
-; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm24, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $48, %zmm11, %zmm3
+; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm28, %xmm2
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm8, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm30
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm13, %xmm2
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm17, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm16, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm31, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm15, %xmm8
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
-; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm8, %xmm6, %xmm9
-; AVX512BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT: vpshufb %xmm8, %xmm6, %xmm13
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
+; AVX512BW-NEXT: vpsrlq $56, %ymm19, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm7, %xmm9
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm27, %xmm9
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm30, %xmm6
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512BW-NEXT: vpsrlq $56, %zmm10, %zmm6
-; AVX512BW-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512BW-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
-; AVX512BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm14, %xmm2
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm17, %xmm6
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm21, %xmm6
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm22, %xmm9
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm25, %xmm6
-; AVX512BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512BW-NEXT: vpshufb %xmm8, %xmm31, %xmm6
-; AVX512BW-NEXT: vpshufb %xmm8, %xmm20, %xmm8
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm11, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm1, %xmm23, %xmm1
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm26, %xmm4
-; AVX512BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
-; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT: vpsrlq $56, %zmm19, %zmm3
-; AVX512BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%rsi)
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%rdx)
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%rcx)
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%r8)
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%r9)
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm8, %xmm6
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512BW-NEXT: vpsrlq $56, %zmm9, %zmm5
+; AVX512BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
+; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm23, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm26, %xmm6
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm28, %xmm5
+; AVX512BW-NEXT: vpshufb %xmm3, %xmm18, %xmm3
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT: vpsrlq $56, %ymm12, %ymm5
+; AVX512BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
+; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT: vpsrlq $56, %zmm11, %zmm2
+; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%rsi)
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%rdx)
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%rcx)
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%r8)
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%r9)
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT: vmovaps %zmm2, (%rax)
+; AVX512BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT: vmovaps %zmm1, (%rax)
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT: vmovdqa64 %zmm30, (%rax)
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax)
-; AVX512BW-NEXT: addq $744, %rsp # imm = 0x2E8
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT: addq $552, %rsp # imm = 0x228
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -16428,678 +16079,621 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
;
; AVX512DQ-BW-LABEL: load_i8_stride8_vf64:
; AVX512DQ-BW: # %bb.0:
-; AVX512DQ-BW-NEXT: subq $744, %rsp # imm = 0x2E8
-; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm1
+; AVX512DQ-BW-NEXT: subq $552, %rsp # imm = 0x228
+; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm10
+; AVX512DQ-BW-NEXT: vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm12
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm0
; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %zmm2
-; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa64 496(%rdi), %xmm24
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm24, %xmm3
+; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 496(%rdi), %xmm27
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm17 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm27, %xmm4
; AVX512DQ-BW-NEXT: vmovdqa64 480(%rdi), %xmm25
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm25, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm26
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm26, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %xmm30
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm30, %xmm6
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm4
-; AVX512DQ-BW-NEXT: vpmovqb %ymm4, %xmm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa64 368(%rdi), %xmm31
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm31, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %xmm27
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm27, %xmm6
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 336(%rdi), %xmm22
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm22, %xmm11
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %xmm9
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm9, %xmm15
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
-; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm11
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm15
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm10, %zmm0, %zmm20
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm25, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm31
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm18 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %xmm28
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm28, %xmm8
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %ymm16
+; AVX512DQ-BW-NEXT: vpmovqb %ymm16, %xmm8
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4],ymm5[5],ymm8[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 368(%rdi), %xmm23
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm23, %xmm8
+; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %xmm24
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm24, %xmm9
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-BW-NEXT: vmovdqa64 336(%rdi), %xmm26
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm26, %xmm14
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %xmm19
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm19, %xmm15
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
+; AVX512DQ-BW-NEXT: vpmovqb %zmm0, %xmm14
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm9
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm13, %zmm0, %zmm21
; AVX512DQ-BW-NEXT: movb $-64, %al
; AVX512DQ-BW-NEXT: kmovd %eax, %k1
-; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa64 240(%rdi), %xmm28
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm28, %xmm7
-; AVX512DQ-BW-NEXT: vmovdqa64 224(%rdi), %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm18, %xmm10
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT: vmovdqa64 208(%rdi), %xmm17
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm17, %xmm10
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm8
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm8, %xmm16
-; AVX512DQ-BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm10
-; AVX512DQ-BW-NEXT: vpmovqb %ymm10, %xmm10
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm10, %xmm16
-; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm14
-; AVX512DQ-BW-NEXT: vpshufb %xmm12, %xmm14, %xmm12
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
-; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm2, %xmm21
-; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm2, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm21 {%k1}
+; AVX512DQ-BW-NEXT: vmovdqa 240(%rdi), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm0, %xmm13
+; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm0, %xmm15
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm6, %xmm15
+; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm14
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm14, %xmm20
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vpmovqb %ymm0, %xmm20
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm20, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm13, %xmm20
+; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm15, %xmm17
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm17[0],xmm20[0],xmm17[1],xmm20[1],xmm17[2],xmm20[2],xmm17[3],xmm20[3]
+; AVX512DQ-BW-NEXT: vmovdqa 80(%rdi), %xmm12
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm12, %xmm22
+; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vpshufb %xmm18, %xmm7, %xmm18
+; AVX512DQ-BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm18[0],xmm22[0],xmm18[1],xmm22[1],xmm18[2],xmm22[2],xmm18[3],xmm22[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpmovqb %zmm10, %xmm2
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm21, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm6
-; AVX512DQ-BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %xmm7
-; AVX512DQ-BW-NEXT: vmovdqa64 400(%rdi), %xmm21
-; AVX512DQ-BW-NEXT: vmovdqa64 416(%rdi), %xmm23
-; AVX512DQ-BW-NEXT: vmovdqa64 432(%rdi), %xmm29
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm19
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm25, %xmm11
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm26, %xmm12
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm26, %xmm24
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm30, %xmm16
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm30, %xmm25
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm29, %xmm24
-; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm23, %xmm25
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm21, %xmm24
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm7, %xmm25
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm24, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm31, %xmm2
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm27, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm22, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm9, %xmm24
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm31
-; AVX512DQ-BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm15, %zmm3
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm15, %zmm22
-; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm28, %xmm9
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm28, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm18, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm17, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm8, %xmm24
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %xmm20
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 176(%rdi), %xmm22
+; AVX512DQ-BW-NEXT: vmovdqa64 416(%rdi), %xmm29
+; AVX512DQ-BW-NEXT: vmovdqa 432(%rdi), %xmm8
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm27, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm27, %xmm10
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm31, %xmm27
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm28, %xmm11
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm28, %xmm28
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm0, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm26, %xmm6, %xmm25
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm6
-; AVX512DQ-BW-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm0, %xmm0
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm6, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm10, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm14, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm15, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm28, %xmm4
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm8, %xmm27
+; AVX512DQ-BW-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm29, %xmm28
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vpsrlq $8, %ymm16, %ymm27
+; AVX512DQ-BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm24, %xmm4
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm5, %zmm3
-; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm26, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm19, %xmm18
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm19, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
+; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm9, %zmm4
+; AVX512DQ-BW-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
+; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm17, %xmm4
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm22, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, %xmm19
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm20, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm22 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vpsrlq $8, %ymm22, %ymm27
+; AVX512DQ-BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm12, %xmm20
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vpsrlq $8, %zmm12, %zmm2
+; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm19, %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm19, %xmm20
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm11, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa %xmm10, %xmm7
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm10, %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm25, %xmm28
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm25, %xmm3
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm12, %xmm25
-; AVX512DQ-BW-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm31, %xmm30
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm31
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm29, %xmm24
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm29, %xmm13
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm23, %xmm30
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, %xmm16
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm21, %xmm18
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm21, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm7, %xmm17
-; AVX512DQ-BW-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm7, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm8, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm29, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm16, %ymm10
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vpsrlq $16, %ymm16, %ymm27
+; AVX512DQ-BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm12, %xmm2
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm22, %zmm6
-; AVX512DQ-BW-NEXT: vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm22, %zmm13
-; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, %xmm25
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm23, %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm16
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm26, %xmm8
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm18, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm9, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm26
-; AVX512DQ-BW-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm21, %xmm0
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm6, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa %xmm6, %xmm11
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm7, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm30, %xmm23, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm19, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm24 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm4
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vpsrlq $16, %ymm22, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm22, %ymm23
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm14, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm13, %xmm19
+; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm15, %xmm22
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm28, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm3
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm5, %zmm3
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm28
+; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm12, %zmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, %zmm6
; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa %xmm7, %xmm15
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm28, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm30, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm31, %xmm28
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm13, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm29, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpsrlq $24, %ymm10, %ymm27
+; AVX512DQ-BW-NEXT: vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm27, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, (%rsp) # 16-byte Spill
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm18, %xmm27
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm14, %xmm17
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm31
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm4
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm23, %ymm12
+; AVX512DQ-BW-NEXT: vpsrlq $24, %ymm23, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm19, %xmm10
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm19, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm26, %xmm24
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm6, %zmm3
+; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm25, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm4, %xmm4
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm30, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm28, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm28, %xmm23
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm24, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm17, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm13, %xmm19
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vpsrlq $32, %ymm30, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa %xmm12, %xmm10
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
-; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm6, %zmm13
-; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm18, %xmm28
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm26, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm22, %xmm21
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm30
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm29, %xmm22
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm18
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm17, %xmm26
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm17, %xmm28
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm9, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm25
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa %xmm7, %xmm9
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm23, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm7, %xmm28
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm31, %xmm11
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT: vpsrlq $32, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm5, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm14, %xmm23
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm15, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm28, %zmm17
-; AVX512DQ-BW-NEXT: vpsrlq $24, %zmm28, %zmm2
+; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm6, %zmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, %zmm7
; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm20, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm28
-; AVX512DQ-BW-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm24, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm26, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, %xmm24
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm18, %xmm2
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa %xmm10, %xmm6
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm27, %xmm7
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm10, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm10, %zmm13
-; AVX512DQ-BW-NEXT: vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm13
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm19, %xmm8, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm5, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm17, %zmm20
-; AVX512DQ-BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm17, %zmm3
-; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm28, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa %xmm8, %xmm13
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm12, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm16, %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm17, %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm23, %xmm6
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm23, %xmm4
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vmovdqa (%rsp), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm24, %xmm19
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, %xmm26
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm19, %xmm31
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm19, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpsrlq $40, %ymm30, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm30, %ymm19
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm6, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm6, %xmm24
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm5, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa (%rsp), %xmm8 # 16-byte Reload
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm30
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
-; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm10, %zmm5
+; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm9, %zmm5
; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm5
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm17
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm18, %xmm23
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm18, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm26, %xmm30
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm25, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm9, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 %xmm11, %xmm18
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm11, %xmm3
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vpsrlq $40, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm20, %zmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm11
+; AVX512DQ-BW-NEXT: vpsrlq $40, %zmm7, %zmm2
; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm28, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm8, %xmm2
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm13, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm17, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm6, %xmm5
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm5
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm26, %xmm19
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm29, %xmm30
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm19, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm24, %xmm3
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm7, %xmm5
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm30, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
-; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm10, %zmm5
+; AVX512DQ-BW-NEXT: vpsrlq $48, %ymm19, %ymm30
+; AVX512DQ-BW-NEXT: vpmovqb %ymm30, %xmm30
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, %xmm30, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm8, %xmm30
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm9, %zmm5
; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm3, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm21, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm22, %xmm19
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm23, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm30
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm9, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm9, %xmm18
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm31, %xmm5
-; AVX512DQ-BW-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm20, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vpsrlq $48, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm11, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm23, %xmm0
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa64 %xmm6, %xmm26
-; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm29, %xmm2
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm19, %zmm2
-; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm24, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm11, %zmm3
+; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm28, %xmm2
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm8, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm30
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm13, %xmm2
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm17, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm6, %xmm4
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm31, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm5
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm5, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm15, %xmm8
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm6, %xmm9
-; AVX512DQ-BW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm6, %xmm13
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
+; AVX512DQ-BW-NEXT: vpsrlq $56, %ymm19, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm24, %xmm5
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm7, %xmm9
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm27, %xmm9
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm30, %xmm6
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm10, %zmm6
-; AVX512DQ-BW-NEXT: vpmovqb %zmm6, %xmm6
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm5, %zmm0, %zmm5
-; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm14, %xmm2
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm17, %xmm6
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm21, %xmm6
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm22, %xmm9
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm25, %xmm6
-; AVX512DQ-BW-NEXT: vpshufb %xmm4, %xmm18, %xmm4
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm31, %xmm6
-; AVX512DQ-BW-NEXT: vpshufb %xmm8, %xmm20, %xmm8
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm11, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm23, %xmm1
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm26, %xmm4
-; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm29, %xmm3
-; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm19, %zmm3
-; AVX512DQ-BW-NEXT: vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rsi)
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rdx)
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rcx)
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%r8)
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%r9)
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm8, %xmm6
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm4, %zmm0, %zmm4
+; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm23, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm26, %xmm6
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm18, %xmm3
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vpsrlq $56, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT: vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm0, %xmm22, %xmm0
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT: vpshufb %xmm2, %xmm24, %xmm2
+; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT: vpsrlq $56, %zmm11, %zmm2
+; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%rsi)
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%rcx)
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%r8)
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%r9)
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT: vmovaps %zmm2, (%rax)
+; AVX512DQ-BW-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT: vmovaps %zmm1, (%rax)
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm30, (%rax)
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rax)
-; AVX512DQ-BW-NEXT: addq $744, %rsp # imm = 0x2E8
+; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-BW-NEXT: addq $552, %rsp # imm = 0x228
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
More information about the llvm-commits
mailing list