[llvm] b911cbd - [X86] Add missing common AVX2 check prefix
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 18 12:25:24 PDT 2022
Author: Simon Pilgrim
Date: 2022-06-18T20:24:57+01:00
New Revision: b911cbdcb9c41fd2d191eb5a9246ae4700968014
URL: https://github.com/llvm/llvm-project/commit/b911cbdcb9c41fd2d191eb5a9246ae4700968014
DIFF: https://github.com/llvm/llvm-project/commit/b911cbdcb9c41fd2d191eb5a9246ae4700968014.diff
LOG: [X86] Add missing common AVX2 check prefix
Added:
Modified:
llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index 54c12589b943..30e5b34ad4b8 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck --check-prefixes=AVX2-SLOW %s
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2-FAST %s
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2-FAST %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck --check-prefixes=AVX2 %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2 %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2 %s
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512
; These patterns are produced by LoopVectorizer for interleaved loads.
@@ -71,41 +71,23 @@ define void @load_i8_stride6_vf2(<12 x i8>* %in.vec, <2 x i8>* %out.vec0, <2 x i
; AVX1-NEXT: vpextrw $0, %xmm0, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: load_i8_stride6_vf2:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm1, (%rsi)
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm2, (%rdx)
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm3, (%rcx)
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm4, (%r8)
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm5, (%r9)
-; AVX2-SLOW-NEXT: vpextrw $0, %xmm0, (%rax)
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: load_i8_stride6_vf2:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpextrw $0, %xmm1, (%rsi)
-; AVX2-FAST-NEXT: vpextrw $0, %xmm2, (%rdx)
-; AVX2-FAST-NEXT: vpextrw $0, %xmm3, (%rcx)
-; AVX2-FAST-NEXT: vpextrw $0, %xmm4, (%r8)
-; AVX2-FAST-NEXT: vpextrw $0, %xmm5, (%r9)
-; AVX2-FAST-NEXT: vpextrw $0, %xmm0, (%rax)
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: load_i8_stride6_vf2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[2,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[3,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpextrw $0, %xmm1, (%rsi)
+; AVX2-NEXT: vpextrw $0, %xmm2, (%rdx)
+; AVX2-NEXT: vpextrw $0, %xmm3, (%rcx)
+; AVX2-NEXT: vpextrw $0, %xmm4, (%r8)
+; AVX2-NEXT: vpextrw $0, %xmm5, (%r9)
+; AVX2-NEXT: vpextrw $0, %xmm0, (%rax)
+; AVX2-NEXT: retq
;
; AVX512-LABEL: load_i8_stride6_vf2:
; AVX512: # %bb.0:
@@ -250,67 +232,36 @@ define void @load_i8_stride6_vf4(<24 x i8>* %in.vec, <4 x i8>* %out.vec0, <4 x i
; AVX1-NEXT: vmovd %xmm0, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: load_i8_stride6_vf4:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-SLOW-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,6,12],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm2, %xmm3, %xmm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm1[3,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[1,7,13],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm1[4,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[2,8,14],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm4, %xmm5, %xmm4
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm1[5,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[3,9,15],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-SLOW-NEXT: vmovd %xmm2, (%rsi)
-; AVX2-SLOW-NEXT: vmovd %xmm3, (%rdx)
-; AVX2-SLOW-NEXT: vmovd %xmm4, (%rcx)
-; AVX2-SLOW-NEXT: vmovd %xmm5, (%r8)
-; AVX2-SLOW-NEXT: vmovd %xmm6, (%r9)
-; AVX2-SLOW-NEXT: vmovd %xmm0, (%rax)
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: load_i8_stride6_vf4:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-FAST-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,6,12],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm2, %xmm3, %xmm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm1[3,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[1,7,13],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm1[4,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[2,8,14],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm4, %xmm5, %xmm4
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm1[5,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[3,9,15],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-FAST-NEXT: vmovd %xmm2, (%rsi)
-; AVX2-FAST-NEXT: vmovd %xmm3, (%rdx)
-; AVX2-FAST-NEXT: vmovd %xmm4, (%rcx)
-; AVX2-FAST-NEXT: vmovd %xmm5, (%r8)
-; AVX2-FAST-NEXT: vmovd %xmm6, (%r9)
-; AVX2-FAST-NEXT: vmovd %xmm0, (%rax)
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: load_i8_stride6_vf4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,6,12],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm1[3,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm0[1,7,13],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm1[4,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[2,8,14],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm4, %xmm5, %xmm4
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm1[5,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[3,9,15],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm5, %xmm6, %xmm5
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[0,6,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,7,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT: vmovd %xmm2, (%rsi)
+; AVX2-NEXT: vmovd %xmm3, (%rdx)
+; AVX2-NEXT: vmovd %xmm4, (%rcx)
+; AVX2-NEXT: vmovd %xmm5, (%r8)
+; AVX2-NEXT: vmovd %xmm6, (%r9)
+; AVX2-NEXT: vmovd %xmm0, (%rax)
+; AVX2-NEXT: retq
;
; AVX512-LABEL: load_i8_stride6_vf4:
; AVX512: # %bb.0:
@@ -561,81 +512,43 @@ define void @load_i8_stride6_vf8(<48 x i8>* %in.vec, <8 x i8>* %out.vec0, <8 x i
; AVX1-NEXT: vmovq %xmm0, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: load_i8_stride6_vf8:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm3[2,8,14],zero,zero,xmm3[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[0,6,12],zero,zero,zero,xmm2[4,10,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm4, %xmm5, %xmm4
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[3,9,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[1,7,13],zero,zero,zero,xmm2[5,11,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm5
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[2,8,14],zero,zero,xmm3[0,6,12,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm7, %xmm6
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm1[0,6,12],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[1,7,13],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vmovq %xmm4, (%rsi)
-; AVX2-SLOW-NEXT: vmovq %xmm2, (%rdx)
-; AVX2-SLOW-NEXT: vmovq %xmm6, (%rcx)
-; AVX2-SLOW-NEXT: vmovq %xmm3, (%r8)
-; AVX2-SLOW-NEXT: vmovq %xmm5, (%r9)
-; AVX2-SLOW-NEXT: vmovq %xmm0, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: load_i8_stride6_vf8:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm3[2,8,14],zero,zero,xmm3[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[0,6,12],zero,zero,zero,xmm2[4,10,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm4, %xmm5, %xmm4
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[3,9,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[1,7,13],zero,zero,zero,xmm2[5,11,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm5
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[2,8,14],zero,zero,xmm3[0,6,12,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm6, %xmm7, %xmm6
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm1[0,6,12],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm7, %xmm5
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[1,7,13],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT: vmovq %xmm4, (%rsi)
-; AVX2-FAST-NEXT: vmovq %xmm2, (%rdx)
-; AVX2-FAST-NEXT: vmovq %xmm6, (%rcx)
-; AVX2-FAST-NEXT: vmovq %xmm3, (%r8)
-; AVX2-FAST-NEXT: vmovq %xmm5, (%r9)
-; AVX2-FAST-NEXT: vmovq %xmm0, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: load_i8_stride6_vf8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7,8,9],ymm1[10],ymm0[11,12],ymm1[13],ymm0[14,15]
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm3[2,8,14],zero,zero,xmm3[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[0,6,12],zero,zero,zero,xmm2[4,10,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm4, %xmm5, %xmm4
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[3,9,15],zero,zero,xmm3[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[1,7,13],zero,zero,zero,xmm2[5,11,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
+; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[2,8,14],zero,zero,xmm3[0,6,12,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm6, %xmm7, %xmm6
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm5, %xmm3, %xmm3
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm1[0,6,12],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm5, %xmm7, %xmm5
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm1[1,7,13],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm4, (%rsi)
+; AVX2-NEXT: vmovq %xmm2, (%rdx)
+; AVX2-NEXT: vmovq %xmm6, (%rcx)
+; AVX2-NEXT: vmovq %xmm3, (%r8)
+; AVX2-NEXT: vmovq %xmm5, (%r9)
+; AVX2-NEXT: vmovq %xmm0, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: load_i8_stride6_vf8:
; AVX512: # %bb.0:
@@ -1083,141 +996,73 @@ define void @load_i8_stride6_vf16(<96 x i8>* %in.vec, <16 x i8>* %out.vec0, <16
; AVX1-NEXT: vmovdqa %xmm0, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: load_i8_stride6_vf16:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm8
-; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm4
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
-; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm8, %ymm4, %ymm5
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm0, %xmm1, %xmm2
-; AVX2-SLOW-NEXT: vmovdqa 80(%rdi), %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[4,10]
-; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm7, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX2-SLOW-NEXT: vpblendvb %xmm11, %xmm2, %xmm3, %xmm9
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm3, %xmm5, %xmm3
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[5,11]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-SLOW-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm10
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-SLOW-NEXT: vpblendvb %ymm3, %ymm4, %ymm8, %ymm3
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm3, %xmm6
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm3[2,8,14],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm2, %xmm5, %xmm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[0,6,12]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm7, %xmm5
-; AVX2-SLOW-NEXT: vpblendvb %xmm11, %xmm2, %xmm5, %xmm12
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[1,7,13]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-SLOW-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm3
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
-; AVX2-SLOW-NEXT: vpblendvb %ymm5, %ymm4, %ymm8, %ymm4
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,xmm5[0,6,12],zero,zero,zero,xmm5[4,10,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm4[4,10],zero,zero,zero,xmm4[2,8,14],zero,zero,xmm4[u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm6, %xmm7, %xmm6
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[2,8,14]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u,u,u,u,u,u,0,6,12],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm7, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[1,7,13],zero,zero,zero,xmm5[5,11,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[5,11],zero,zero,zero,xmm4[3,9,15],zero,zero,xmm4[u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm5, %xmm4, %xmm4
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[3,9,15]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,u,u,1,7,13],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
-; AVX2-SLOW-NEXT: vmovdqa %xmm9, (%rsi)
-; AVX2-SLOW-NEXT: vmovdqa %xmm10, (%rdx)
-; AVX2-SLOW-NEXT: vmovdqa %xmm12, (%rcx)
-; AVX2-SLOW-NEXT: vmovdqa %xmm3, (%r8)
-; AVX2-SLOW-NEXT: vmovdqa %xmm2, (%r9)
-; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: load_i8_stride6_vf16:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm8
-; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm4
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
-; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm8, %ymm4, %ymm5
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm0, %xmm1, %xmm2
-; AVX2-FAST-NEXT: vmovdqa 80(%rdi), %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[4,10]
-; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
-; AVX2-FAST-NEXT: vpor %xmm7, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX2-FAST-NEXT: vpblendvb %xmm11, %xmm2, %xmm3, %xmm9
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm3, %xmm5, %xmm3
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[5,11]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-FAST-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm10
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-FAST-NEXT: vpblendvb %ymm3, %ymm4, %ymm8, %ymm3
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm3, %xmm6
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm3[2,8,14],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm2, %xmm5, %xmm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[0,6,12]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm7, %xmm5
-; AVX2-FAST-NEXT: vpblendvb %xmm11, %xmm2, %xmm5, %xmm12
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[1,7,13]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm6, %xmm5
-; AVX2-FAST-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm3
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
-; AVX2-FAST-NEXT: vpblendvb %ymm5, %ymm4, %ymm8, %ymm4
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,xmm5[0,6,12],zero,zero,zero,xmm5[4,10,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm4[4,10],zero,zero,zero,xmm4[2,8,14],zero,zero,xmm4[u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm6, %xmm7, %xmm6
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[2,8,14]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u,u,u,u,u,u,0,6,12],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %xmm7, %xmm2, %xmm2
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[1,7,13],zero,zero,zero,xmm5[5,11,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[5,11],zero,zero,zero,xmm4[3,9,15],zero,zero,xmm4[u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm5, %xmm4, %xmm4
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[3,9,15]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,u,u,1,7,13],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
-; AVX2-FAST-NEXT: vmovdqa %xmm9, (%rsi)
-; AVX2-FAST-NEXT: vmovdqa %xmm10, (%rdx)
-; AVX2-FAST-NEXT: vmovdqa %xmm12, (%rcx)
-; AVX2-FAST-NEXT: vmovdqa %xmm3, (%r8)
-; AVX2-FAST-NEXT: vmovdqa %xmm2, (%r9)
-; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: load_i8_stride6_vf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %ymm8
+; AVX2-NEXT: vmovdqa 32(%rdi), %ymm4
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
+; AVX2-NEXT: vpblendvb %ymm0, %ymm8, %ymm4, %ymm5
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
+; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm2
+; AVX2-NEXT: vmovdqa 80(%rdi), %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[4,10]
+; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
+; AVX2-NEXT: vpor %xmm7, %xmm3, %xmm3
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX2-NEXT: vpblendvb %xmm11, %xmm2, %xmm3, %xmm9
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm3, %xmm5, %xmm3
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[5,11]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
+; AVX2-NEXT: vpor %xmm5, %xmm6, %xmm5
+; AVX2-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm10
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
+; AVX2-NEXT: vpblendvb %ymm3, %ymm4, %ymm8, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm6
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm3[2,8,14],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm2, %xmm5, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[0,6,12]
+; AVX2-NEXT: vpor %xmm5, %xmm7, %xmm5
+; AVX2-NEXT: vpblendvb %xmm11, %xmm2, %xmm5, %xmm12
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[3,9,15],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm5, %xmm3, %xmm3
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm0[1,7,13]
+; AVX2-NEXT: vpor %xmm5, %xmm6, %xmm5
+; AVX2-NEXT: vpblendvb %xmm11, %xmm3, %xmm5, %xmm3
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
+; AVX2-NEXT: vpblendvb %ymm5, %ymm4, %ymm8, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
+; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,xmm5[0,6,12],zero,zero,zero,xmm5[4,10,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm4[4,10],zero,zero,zero,xmm4[2,8,14],zero,zero,xmm4[u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm6, %xmm7, %xmm6
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[2,8,14]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u,u,u,u,u,u,0,6,12],zero,zero,zero
+; AVX2-NEXT: vpor %xmm7, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[1,7,13],zero,zero,zero,xmm5[5,11,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[5,11],zero,zero,zero,xmm4[3,9,15],zero,zero,xmm4[u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm0[3,9,15]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,u,u,1,7,13],zero,zero,zero
+; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3,4],xmm0[5,6,7]
+; AVX2-NEXT: vmovdqa %xmm9, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm10, (%rdx)
+; AVX2-NEXT: vmovdqa %xmm12, (%rcx)
+; AVX2-NEXT: vmovdqa %xmm3, (%r8)
+; AVX2-NEXT: vmovdqa %xmm2, (%r9)
+; AVX2-NEXT: vmovdqa %xmm0, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: load_i8_stride6_vf16:
; AVX512: # %bb.0:
@@ -2290,241 +2135,123 @@ define void @load_i8_stride6_vf32(<192 x i8>* %in.vec, <32 x i8>* %out.vec0, <32
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: load_i8_stride6_vf32:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %ymm14
-; AVX2-SLOW-NEXT: vmovdqa 32(%rdi), %ymm13
-; AVX2-SLOW-NEXT: vmovdqa 64(%rdi), %ymm0
-; AVX2-SLOW-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX2-SLOW-NEXT: vmovdqa 160(%rdi), %ymm12
-; AVX2-SLOW-NEXT: vmovdqa 128(%rdi), %ymm6
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
-; AVX2-SLOW-NEXT: vpblendvb %ymm9, %ymm14, %ymm13, %ymm7
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm7, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,8,14],zero,zero,xmm1[0,6,12,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm4[2,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,ymm3[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,10],zero,zero,zero,ymm3[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[20,26]
-; AVX2-SLOW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm0[0,1],ymm4[0,1]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,8,14],zero,zero,ymm4[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[18,24,30],zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm0, %ymm8, %ymm0
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX2-SLOW-NEXT: vpblendvb %ymm8, %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm7[1,7,13],zero,zero,zero,xmm7[5,11],zero,zero,zero,xmm7[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm2, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[5,11],zero,zero,zero,ymm3[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[21,27]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[3,9,15],zero,zero,ymm4[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[19,25,31],zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm2, %ymm7, %ymm2
-; AVX2-SLOW-NEXT: vpblendvb %ymm8, %ymm1, %ymm2, %ymm0
-; AVX2-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm13, %ymm14, %ymm2
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm2, %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm2[2,8,14],zero,zero,xmm2[0,6,12],zero,zero,zero,xmm2[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm7, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,ymm3[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,6,12],zero,zero,zero,ymm3[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,22,28]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm4[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[4,10],zero,zero,zero,ymm4[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[20,26],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm10, %ymm7
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm15 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
-; AVX2-SLOW-NEXT: vpblendvb %ymm9, %ymm6, %ymm12, %ymm9
-; AVX2-SLOW-NEXT: vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[3,9,15],zero,zero,xmm2[1,7,13],zero,zero,zero,xmm2[u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,7,13],zero,zero,zero,ymm3[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[17,23,29]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[5,11],zero,zero,zero,ymm4[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[21,27],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm2, %ymm7, %ymm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,0,6,12],zero,zero,zero,xmm9[4,10],zero,zero,zero
-; AVX2-SLOW-NEXT: vpblendvb %ymm8, %ymm0, %ymm2, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm9, %xmm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
-; AVX2-SLOW-NEXT: vpor %xmm7, %xmm5, %xmm5
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
-; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm1, %ymm5, %ymm8
-; AVX2-SLOW-NEXT: vpblendvb %ymm15, %ymm12, %ymm6, %ymm10
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm9[u,u,u,u,u,1,7,13],zero,zero,zero,xmm9[5,11],zero,zero,zero
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm0, %ymm1, %ymm9
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-SLOW-NEXT: vpblendvb %ymm0, %ymm12, %ymm6, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[4,10],zero,zero,zero,xmm1[2,8,14]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,2,8,14],zero,zero,xmm0[0,6,12],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm2, %xmm5, %xmm12
-; AVX2-SLOW-NEXT: vpblendvb %ymm15, %ymm13, %ymm14, %ymm5
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = zero,zero,xmm6[0,6,12],zero,zero,zero,xmm6[4,10,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm2 = xmm5[4,10],zero,zero,zero,xmm5[2,8,14],zero,zero,xmm5[u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm7, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[0,6,12],zero,zero,zero,ymm4[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,22,28],zero,zero,zero
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,ymm3[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,8,14],zero,zero,ymm3[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[18,24,30]
-; AVX2-SLOW-NEXT: vpor %ymm7, %ymm13, %ymm7
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm7[5,6,7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm7
-; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm2, %ymm7, %ymm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[5,11],zero,zero,zero,xmm1[3,9,15]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,3,9,15],zero,zero,xmm0[1,7,13],zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm6[1,7,13],zero,zero,zero,xmm6[5,11,u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[5,11],zero,zero,zero,xmm5[3,9,15],zero,zero,xmm5[u,u,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm5, %xmm1
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[1,7,13],zero,zero,zero,ymm4[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[17,23,29],zero,zero,zero
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,9,15],zero,zero,ymm3[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,25,31]
-; AVX2-SLOW-NEXT: vpor %ymm4, %ymm3, %ymm3
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm3[5,6,7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm10, %xmm3
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,4,10],zero,zero,zero,xmm10[2,8,14],zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm4, %xmm1
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7],ymm4[8,9,10],ymm1[11,12,13,14,15]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[5,11]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,5,11],zero,zero,zero,xmm10[3,9,15],zero,zero
-; AVX2-SLOW-NEXT: vpor %xmm3, %xmm4, %xmm3
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX2-SLOW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rsi)
-; AVX2-SLOW-NEXT: vmovdqa %ymm3, (%rdx)
-; AVX2-SLOW-NEXT: vmovdqa %ymm8, (%rcx)
-; AVX2-SLOW-NEXT: vmovdqa %ymm9, (%r8)
-; AVX2-SLOW-NEXT: vmovdqa %ymm2, (%r9)
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa %ymm0, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: load_i8_stride6_vf32:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %ymm14
-; AVX2-FAST-NEXT: vmovdqa 32(%rdi), %ymm13
-; AVX2-FAST-NEXT: vmovdqa 64(%rdi), %ymm0
-; AVX2-FAST-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX2-FAST-NEXT: vmovdqa 160(%rdi), %ymm12
-; AVX2-FAST-NEXT: vmovdqa 128(%rdi), %ymm6
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm9 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
-; AVX2-FAST-NEXT: vpblendvb %ymm9, %ymm14, %ymm13, %ymm7
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm7, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,8,14],zero,zero,xmm1[0,6,12,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm4[2,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,ymm3[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,10],zero,zero,zero,ymm3[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[20,26]
-; AVX2-FAST-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm0[0,1],ymm4[0,1]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,8,14],zero,zero,ymm4[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[18,24,30],zero,zero
-; AVX2-FAST-NEXT: vpor %ymm0, %ymm8, %ymm0
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX2-FAST-NEXT: vpblendvb %ymm8, %ymm2, %ymm0, %ymm0
-; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm7[1,7,13],zero,zero,zero,xmm7[5,11],zero,zero,zero,xmm7[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm2, %xmm1, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[5,11],zero,zero,zero,ymm3[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[21,27]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[3,9,15],zero,zero,ymm4[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[19,25,31],zero,zero
-; AVX2-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2
-; AVX2-FAST-NEXT: vpblendvb %ymm8, %ymm1, %ymm2, %ymm0
-; AVX2-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm13, %ymm14, %ymm2
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm2, %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm2[2,8,14],zero,zero,xmm2[0,6,12],zero,zero,zero,xmm2[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm7, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,ymm3[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,6,12],zero,zero,zero,ymm3[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,22,28]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm10 = ymm4[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[4,10],zero,zero,zero,ymm4[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[20,26],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %ymm7, %ymm10, %ymm7
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm15 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
-; AVX2-FAST-NEXT: vpblendvb %ymm9, %ymm6, %ymm12, %ymm9
-; AVX2-FAST-NEXT: vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[3,9,15],zero,zero,xmm2[1,7,13],zero,zero,zero,xmm2[u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,7,13],zero,zero,zero,ymm3[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[17,23,29]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[5,11],zero,zero,zero,ymm4[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[21,27],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,0,6,12],zero,zero,zero,xmm9[4,10],zero,zero,zero
-; AVX2-FAST-NEXT: vpblendvb %ymm8, %ymm0, %ymm2, %ymm0
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm9, %xmm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
-; AVX2-FAST-NEXT: vpor %xmm7, %xmm5, %xmm5
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
-; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm1, %ymm5, %ymm8
-; AVX2-FAST-NEXT: vpblendvb %ymm15, %ymm12, %ymm6, %ymm10
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm9[u,u,u,u,u,1,7,13],zero,zero,zero,xmm9[5,11],zero,zero,zero
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm2, %xmm1
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm0, %ymm1, %ymm9
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
-; AVX2-FAST-NEXT: vpblendvb %ymm0, %ymm12, %ymm6, %ymm0
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[4,10],zero,zero,zero,xmm1[2,8,14]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,2,8,14],zero,zero,xmm0[0,6,12],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %xmm2, %xmm5, %xmm12
-; AVX2-FAST-NEXT: vpblendvb %ymm15, %ymm13, %ymm14, %ymm5
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm5, %xmm6
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm7 = zero,zero,xmm6[0,6,12],zero,zero,zero,xmm6[4,10,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm5[4,10],zero,zero,zero,xmm5[2,8,14],zero,zero,xmm5[u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm7, %xmm2, %xmm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[0,6,12],zero,zero,zero,ymm4[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,22,28],zero,zero,zero
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,ymm3[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,8,14],zero,zero,ymm3[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[18,24,30]
-; AVX2-FAST-NEXT: vpor %ymm7, %ymm13, %ymm7
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm7[5,6,7]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm7
-; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm2, %ymm7, %ymm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[5,11],zero,zero,zero,xmm1[3,9,15]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,3,9,15],zero,zero,xmm0[1,7,13],zero,zero,zero
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm6[1,7,13],zero,zero,zero,xmm6[5,11,u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[5,11],zero,zero,zero,xmm5[3,9,15],zero,zero,xmm5[u,u,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm5, %xmm1
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[1,7,13],zero,zero,zero,ymm4[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[17,23,29],zero,zero,zero
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,9,15],zero,zero,ymm3[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,25,31]
-; AVX2-FAST-NEXT: vpor %ymm4, %ymm3, %ymm3
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm3[5,6,7]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm10, %xmm3
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX2-FAST-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,4,10],zero,zero,zero,xmm10[2,8,14],zero,zero
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm4, %xmm1
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7],ymm4[8,9,10],ymm1[11,12,13,14,15]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[5,11]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,5,11],zero,zero,zero,xmm10[3,9,15],zero,zero
-; AVX2-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX2-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT: vmovdqa %ymm1, (%rsi)
-; AVX2-FAST-NEXT: vmovdqa %ymm3, (%rdx)
-; AVX2-FAST-NEXT: vmovdqa %ymm8, (%rcx)
-; AVX2-FAST-NEXT: vmovdqa %ymm9, (%r8)
-; AVX2-FAST-NEXT: vmovdqa %ymm2, (%r9)
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa %ymm0, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: load_i8_stride6_vf32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %ymm14
+; AVX2-NEXT: vmovdqa 32(%rdi), %ymm13
+; AVX2-NEXT: vmovdqa 64(%rdi), %ymm0
+; AVX2-NEXT: vmovdqa 96(%rdi), %ymm4
+; AVX2-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX2-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm9 = <255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255,u,u,0,0,255,255>
+; AVX2-NEXT: vpblendvb %ymm9, %ymm14, %ymm13, %ymm7
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
+; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm1[2,8,14],zero,zero,xmm1[0,6,12,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm4[2,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm8 = zero,zero,zero,ymm3[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,10],zero,zero,zero,ymm3[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[20,26]
+; AVX2-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm0[0,1],ymm4[0,1]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,8,14],zero,zero,ymm4[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[18,24,30],zero,zero
+; AVX2-NEXT: vpor %ymm0, %ymm8, %ymm0
+; AVX2-NEXT: vmovdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX2-NEXT: vpblendvb %ymm8, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm7[1,7,13],zero,zero,zero,xmm7[5,11],zero,zero,zero,xmm7[u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[5,11],zero,zero,zero,ymm3[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[21,27]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[3,9,15],zero,zero,ymm4[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[19,25,31],zero,zero
+; AVX2-NEXT: vpor %ymm2, %ymm7, %ymm2
+; AVX2-NEXT: vpblendvb %ymm8, %ymm1, %ymm2, %ymm0
+; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
+; AVX2-NEXT: vpblendvb %ymm0, %ymm13, %ymm14, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm0[4,10],zero,zero,zero,xmm0[2,8,14,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm2[2,8,14],zero,zero,xmm2[0,6,12],zero,zero,zero,xmm2[u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm1, %xmm7, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,ymm3[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,6,12],zero,zero,zero,ymm3[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,22,28]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm10 = ymm4[2,8,14],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[4,10],zero,zero,zero,ymm4[18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[20,26],zero,zero,zero
+; AVX2-NEXT: vpor %ymm7, %ymm10, %ymm7
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm15 = <u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u>
+; AVX2-NEXT: vpblendvb %ymm9, %ymm6, %ymm12, %ymm9
+; AVX2-NEXT: vpblendvb %ymm8, %ymm1, %ymm7, %ymm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm0[5,11],zero,zero,zero,xmm0[3,9,15,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[3,9,15],zero,zero,xmm2[1,7,13],zero,zero,zero,xmm2[u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm3[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,7,13],zero,zero,zero,ymm3[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[17,23,29]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[3,9,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[5,11],zero,zero,zero,ymm4[19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[21,27],zero,zero,zero
+; AVX2-NEXT: vpor %ymm2, %ymm7, %ymm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = xmm9[u,u,u,u,u,0,6,12],zero,zero,zero,xmm9[4,10],zero,zero,zero
+; AVX2-NEXT: vpblendvb %ymm8, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
+; AVX2-NEXT: vpor %xmm7, %xmm5, %xmm5
+; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpblendvb %ymm11, %ymm1, %ymm5, %ymm8
+; AVX2-NEXT: vpblendvb %ymm15, %ymm12, %ymm6, %ymm10
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm9[u,u,u,u,u,1,7,13],zero,zero,zero,xmm9[5,11],zero,zero,zero
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
+; AVX2-NEXT: vpor %xmm1, %xmm2, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX2-NEXT: vpblendvb %ymm11, %ymm0, %ymm1, %ymm9
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = <255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255,0,0,u,u,255,255>
+; AVX2-NEXT: vpblendvb %ymm0, %ymm12, %ymm6, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[4,10],zero,zero,zero,xmm1[2,8,14]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,2,8,14],zero,zero,xmm0[0,6,12],zero,zero,zero
+; AVX2-NEXT: vpor %xmm2, %xmm5, %xmm12
+; AVX2-NEXT: vpblendvb %ymm15, %ymm13, %ymm14, %ymm5
+; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6
+; AVX2-NEXT: vpshufb {{.*#+}} xmm7 = zero,zero,xmm6[0,6,12],zero,zero,zero,xmm6[4,10,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm5[4,10],zero,zero,zero,xmm5[2,8,14],zero,zero,xmm5[u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm7, %xmm2, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[4,10],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[0,6,12],zero,zero,zero,ymm4[20,26],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,22,28],zero,zero,zero
+; AVX2-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,ymm3[0,6,12],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,8,14],zero,zero,ymm3[16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[18,24,30]
+; AVX2-NEXT: vpor %ymm7, %ymm13, %ymm7
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm7[5,6,7]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
+; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm7
+; AVX2-NEXT: vpblendvb %ymm11, %ymm2, %ymm7, %ymm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[5,11],zero,zero,zero,xmm1[3,9,15]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,3,9,15],zero,zero,xmm0[1,7,13],zero,zero,zero
+; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm6[1,7,13],zero,zero,zero,xmm6[5,11,u,u,u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[5,11],zero,zero,zero,xmm5[3,9,15],zero,zero,xmm5[u,u,u,u,u,u]
+; AVX2-NEXT: vpor %xmm1, %xmm5, %xmm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[5,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[1,7,13],zero,zero,zero,ymm4[21,27],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[17,23,29],zero,zero,zero
+; AVX2-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[1,7,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,9,15],zero,zero,ymm3[17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,25,31]
+; AVX2-NEXT: vpor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm3[5,6,7]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-NEXT: vextracti128 $1, %ymm10, %xmm3
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX2-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,4,10],zero,zero,zero,xmm10[2,8,14],zero,zero
+; AVX2-NEXT: vpor %xmm1, %xmm4, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3,4,5,6,7],ymm4[8,9,10],ymm1[11,12,13,14,15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u],zero,zero,xmm3[1,7,13],zero,zero,zero,xmm3[5,11]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm10[u,u,u,u,u,u,5,11],zero,zero,zero,xmm10[3,9,15],zero,zero
+; AVX2-NEXT: vpor %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-NEXT: vmovdqa %ymm1, (%rsi)
+; AVX2-NEXT: vmovdqa %ymm3, (%rdx)
+; AVX2-NEXT: vmovdqa %ymm8, (%rcx)
+; AVX2-NEXT: vmovdqa %ymm9, (%r8)
+; AVX2-NEXT: vmovdqa %ymm2, (%r9)
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa %ymm0, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: load_i8_stride6_vf32:
; AVX512: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index 92a3cfa79d36..d0204ff823aa 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck --check-prefixes=AVX2-SLOW %s
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2-FAST %s
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2-FAST %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck --check-prefixes=AVX2,AVX2-SLOW %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2,AVX2-FAST %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck --check-prefixes=AVX2,AVX2-FAST %s
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512
; These patterns are produced by LoopVectorizer for interleaved stores.
@@ -60,39 +60,22 @@ define void @store_i8_stride6_vf2(<2 x i8>* %in.vecptr0, <2 x i8>* %in.vecptr1,
; AVX1-NEXT: vmovq %xmm0, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: store_i8_stride6_vf2:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-SLOW-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-SLOW-NEXT: vmovdqa (%r9), %xmm2
-; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX2-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,1],zero,xmm0[2,6,10,14,3],zero,xmm0[u,u,u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm2[0],zero,zero,zero,zero,zero,xmm2[1,u,u,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpextrd $2, %xmm0, 8(%rax)
-; AVX2-SLOW-NEXT: vmovq %xmm0, (%rax)
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: store_i8_stride6_vf2:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-FAST-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-FAST-NEXT: vmovdqa (%r9), %xmm2
-; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX2-FAST-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,1],zero,xmm0[2,6,10,14,3],zero,xmm0[u,u,u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm2[0],zero,zero,zero,zero,zero,xmm2[1,u,u,u,u]
-; AVX2-FAST-NEXT: vpor %xmm1, %xmm0, %xmm0
-; AVX2-FAST-NEXT: vpextrd $2, %xmm0, 8(%rax)
-; AVX2-FAST-NEXT: vmovq %xmm0, (%rax)
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: store_i8_stride6_vf2:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa (%rdx), %xmm1
+; AVX2-NEXT: vmovdqa (%r9), %xmm2
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,1],zero,xmm0[2,6,10,14,3],zero,xmm0[u,u,u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm2[0],zero,zero,zero,zero,zero,xmm2[1,u,u,u,u]
+; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpextrd $2, %xmm0, 8(%rax)
+; AVX2-NEXT: vmovq %xmm0, (%rax)
+; AVX2-NEXT: retq
;
; AVX512-LABEL: store_i8_stride6_vf2:
; AVX512: # %bb.0:
@@ -199,47 +182,26 @@ define void @store_i8_stride6_vf4(<4 x i8>* %in.vecptr0, <4 x i8>* %in.vecptr1,
; AVX1-NEXT: vmovdqa %xmm2, (%rax)
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: store_i8_stride6_vf4:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-SLOW-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-SLOW-NEXT: vmovdqa (%r8), %xmm2
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,4,8,12],zero,zero,ymm0[1,5,9,13],zero,zero,ymm0[2,6,10,14,18,22],zero,zero,zero,zero,ymm0[19,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,4],zero,zero,zero,zero,ymm0[1,5],zero,zero,zero,zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX2-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-SLOW-NEXT: vmovq %xmm1, 16(%rax)
-; AVX2-SLOW-NEXT: vmovdqa %xmm0, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: store_i8_stride6_vf4:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-FAST-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-FAST-NEXT: vmovdqa (%r8), %xmm2
-; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
-; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX2-FAST-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,4,8,12],zero,zero,ymm0[1,5,9,13],zero,zero,ymm0[2,6,10,14,18,22],zero,zero,zero,zero,ymm0[19,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,4],zero,zero,zero,zero,ymm0[1,5],zero,zero,zero,zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-FAST-NEXT: vmovq %xmm1, 16(%rax)
-; AVX2-FAST-NEXT: vmovdqa %xmm0, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: store_i8_stride6_vf4:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa (%rdx), %xmm1
+; AVX2-NEXT: vmovdqa (%r8), %xmm2
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,4,8,12],zero,zero,ymm0[1,5,9,13],zero,zero,ymm0[2,6,10,14,18,22],zero,zero,zero,zero,ymm0[19,23],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,4],zero,zero,zero,zero,ymm0[1,5],zero,zero,zero,zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vmovq %xmm1, 16(%rax)
+; AVX2-NEXT: vmovdqa %xmm0, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: store_i8_stride6_vf4:
; AVX512: # %bb.0:
@@ -379,67 +341,36 @@ define void @store_i8_stride6_vf8(<8 x i8>* %in.vecptr0, <8 x i8>* %in.vecptr1,
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: store_i8_stride6_vf8:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX2-SLOW-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero
-; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u]
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14],ymm2[15]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[2,10,1,9,0,8,3,11,u,u,u,u,4,12,u,u]
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,0,1]
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
-; AVX2-SLOW-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm2
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[u,u,6,14],zero,zero,xmm0[u,u,7,15],zero,zero,xmm0[u,u]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[5,13,u,u],zero,zero,xmm1[6,14,u,u],zero,zero,xmm1[7,15,u,u]
-; AVX2-SLOW-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
-; AVX2-SLOW-NEXT: vmovdqa %xmm0, 32(%rax)
-; AVX2-SLOW-NEXT: vmovdqa %ymm2, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: store_i8_stride6_vf8:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX2-FAST-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero
-; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u]
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29]
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14],ymm2[15]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[2,10,1,9,0,8,3,11,u,u,u,u,4,12,u,u]
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,0,1]
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
-; AVX2-FAST-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[u,u,6,14],zero,zero,xmm0[u,u,7,15],zero,zero,xmm0[u,u]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[5,13,u,u],zero,zero,xmm1[6,14,u,u],zero,zero,xmm1[7,15,u,u]
-; AVX2-FAST-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15]
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
-; AVX2-FAST-NEXT: vmovdqa %xmm0, 32(%rax)
-; AVX2-FAST-NEXT: vmovdqa %ymm2, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: store_i8_stride6_vf8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX2-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2
+; AVX2-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; AVX2-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = ymm2[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14],ymm2[15]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm3[2,10,1,9,0,8,3,11,u,u,u,u,4,12,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,0,1]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
+; AVX2-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[u,u,6,14],zero,zero,xmm0[u,u,7,15],zero,zero,xmm0[u,u]
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[5,13,u,u],zero,zero,xmm1[6,14,u,u],zero,zero,xmm1[7,15,u,u]
+; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm3[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]
+; AVX2-NEXT: vmovdqa %xmm0, 32(%rax)
+; AVX2-NEXT: vmovdqa %ymm2, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: store_i8_stride6_vf8:
; AVX512: # %bb.0:
@@ -698,89 +629,47 @@ define void @store_i8_stride6_vf16(<16 x i8>* %in.vecptr0, <16 x i8>* %in.vecptr
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-SLOW-LABEL: store_i8_stride6_vf16:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-SLOW-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-SLOW-NEXT: vmovdqa (%r8), %xmm2
-; AVX2-SLOW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX2-SLOW-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,0,2]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,ymm3[2,10],zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,ymm3[20,28],zero,zero,zero,zero
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,zero,ymm4[1,9],zero,zero,zero,zero,ymm4[2,10],zero,zero,zero,zero,ymm4[19,27],zero,zero,zero,zero,ymm4[20,28],zero,zero,zero,zero,ymm4[21,29]
-; AVX2-SLOW-NEXT: vpor %ymm3, %ymm4, %ymm3
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,0,2]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u]
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
-; AVX2-SLOW-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[6,14],zero,zero,zero,zero,ymm4[7,15],zero,zero,zero,zero,ymm4[16,24],zero,zero,zero,zero,ymm4[17,25],zero,zero,zero,zero,ymm4[18,26],zero,zero
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm1[0,2,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[5,13],zero,zero,zero,zero,ymm5[6,14],zero,zero,zero,zero,ymm5[7,15],zero,zero,zero,zero,ymm5[16,24],zero,zero,zero,zero,ymm5[17,25],zero,zero,zero,zero,ymm5[18,26]
-; AVX2-SLOW-NEXT: vpor %ymm4, %ymm5, %ymm4
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15,u,u,u,u,16,24,u,u,u,u,17,25,u,u,u,u]
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm6 = [255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255]
-; AVX2-SLOW-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm4
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[3,11],zero,zero,zero,zero,ymm1[4,12],zero,zero,zero,zero,ymm1[21,29],zero,zero,zero,zero,ymm1[22,30],zero,zero,zero,zero,ymm1[23,31],zero,zero
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,ymm0[3,11],zero,zero,zero,zero,ymm0[4,12],zero,zero,zero,zero,ymm0[5,13],zero,zero,zero,zero,ymm0[22,30],zero,zero,zero,zero,ymm0[23,31],zero,zero,zero,zero
-; AVX2-SLOW-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]
-; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[2,10,u,u,u,u,3,11,u,u,u,u,4,12,u,u,u,u,21,29,u,u,u,u,22,30,u,u,u,u,23,31]
-; AVX2-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0]
-; AVX2-SLOW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-SLOW-NEXT: vmovdqa %ymm0, 64(%rax)
-; AVX2-SLOW-NEXT: vmovdqa %ymm4, 32(%rax)
-; AVX2-SLOW-NEXT: vmovdqa %ymm3, (%rax)
-; AVX2-SLOW-NEXT: vzeroupper
-; AVX2-SLOW-NEXT: retq
-;
-; AVX2-FAST-LABEL: store_i8_stride6_vf16:
-; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-FAST-NEXT: vmovdqa (%rdx), %xmm1
-; AVX2-FAST-NEXT: vmovdqa (%r8), %xmm2
-; AVX2-FAST-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0
-; AVX2-FAST-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX2-FAST-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,0,2]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,ymm3[2,10],zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,ymm3[20,28],zero,zero,zero,zero
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,zero,ymm4[1,9],zero,zero,zero,zero,ymm4[2,10],zero,zero,zero,zero,ymm4[19,27],zero,zero,zero,zero,ymm4[20,28],zero,zero,zero,zero,ymm4[21,29]
-; AVX2-FAST-NEXT: vpor %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,0,2]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u]
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
-; AVX2-FAST-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[6,14],zero,zero,zero,zero,ymm4[7,15],zero,zero,zero,zero,ymm4[16,24],zero,zero,zero,zero,ymm4[17,25],zero,zero,zero,zero,ymm4[18,26],zero,zero
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm5 = ymm1[0,2,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[5,13],zero,zero,zero,zero,ymm5[6,14],zero,zero,zero,zero,ymm5[7,15],zero,zero,zero,zero,ymm5[16,24],zero,zero,zero,zero,ymm5[17,25],zero,zero,zero,zero,ymm5[18,26]
-; AVX2-FAST-NEXT: vpor %ymm4, %ymm5, %ymm4
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15,u,u,u,u,16,24,u,u,u,u,17,25,u,u,u,u]
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm6 = [255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255]
-; AVX2-FAST-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm4
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[3,11],zero,zero,zero,zero,ymm1[4,12],zero,zero,zero,zero,ymm1[21,29],zero,zero,zero,zero,ymm1[22,30],zero,zero,zero,zero,ymm1[23,31],zero,zero
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,ymm0[3,11],zero,zero,zero,zero,ymm0[4,12],zero,zero,zero,zero,ymm0[5,13],zero,zero,zero,zero,ymm0[22,30],zero,zero,zero,zero,ymm0[23,31],zero,zero,zero,zero
-; AVX2-FAST-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]
-; AVX2-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[2,10,u,u,u,u,3,11,u,u,u,u,4,12,u,u,u,u,21,29,u,u,u,u,22,30,u,u,u,u,23,31]
-; AVX2-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0]
-; AVX2-FAST-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-FAST-NEXT: vmovdqa %ymm0, 64(%rax)
-; AVX2-FAST-NEXT: vmovdqa %ymm4, 32(%rax)
-; AVX2-FAST-NEXT: vmovdqa %ymm3, (%rax)
-; AVX2-FAST-NEXT: vzeroupper
-; AVX2-FAST-NEXT: retq
+; AVX2-LABEL: store_i8_stride6_vf16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa (%rdx), %xmm1
+; AVX2-NEXT: vmovdqa (%r8), %xmm2
+; AVX2-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0
+; AVX2-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1
+; AVX2-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,0,2]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,ymm3[2,10],zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,ymm3[20,28],zero,zero,zero,zero
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[0,8],zero,zero,zero,zero,ymm4[1,9],zero,zero,zero,zero,ymm4[2,10],zero,zero,zero,zero,ymm4[19,27],zero,zero,zero,zero,ymm4[20,28],zero,zero,zero,zero,ymm4[21,29]
+; AVX2-NEXT: vpor %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,0,2]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255]
+; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[6,14],zero,zero,zero,zero,ymm4[7,15],zero,zero,zero,zero,ymm4[16,24],zero,zero,zero,zero,ymm4[17,25],zero,zero,zero,zero,ymm4[18,26],zero,zero
+; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[5,13],zero,zero,zero,zero,ymm5[6,14],zero,zero,zero,zero,ymm5[7,15],zero,zero,zero,zero,ymm5[16,24],zero,zero,zero,zero,ymm5[17,25],zero,zero,zero,zero,ymm5[18,26]
+; AVX2-NEXT: vpor %ymm4, %ymm5, %ymm4
+; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,5,13,u,u,u,u,6,14,u,u,u,u,7,15,u,u,u,u,16,24,u,u,u,u,17,25,u,u,u,u]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255]
+; AVX2-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm4
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,ymm1[3,11],zero,zero,zero,zero,ymm1[4,12],zero,zero,zero,zero,ymm1[21,29],zero,zero,zero,zero,ymm1[22,30],zero,zero,zero,zero,ymm1[23,31],zero,zero
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,ymm0[3,11],zero,zero,zero,zero,ymm0[4,12],zero,zero,zero,zero,ymm0[5,13],zero,zero,zero,zero,ymm0[22,30],zero,zero,zero,zero,ymm0[23,31],zero,zero,zero,zero
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[2,10,u,u,u,u,3,11,u,u,u,u,4,12,u,u,u,u,21,29,u,u,u,u,22,30,u,u,u,u,23,31]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0,255,255,255,255,0,0]
+; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vmovdqa %ymm0, 64(%rax)
+; AVX2-NEXT: vmovdqa %ymm4, 32(%rax)
+; AVX2-NEXT: vmovdqa %ymm3, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: store_i8_stride6_vf16:
; AVX512: # %bb.0:
More information about the llvm-commits
mailing list